{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Variable\n",
    "import torch as t"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sigmoid + MSE Cost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Net(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "        \n",
    "        self.fc1 = nn.Linear(28*28, 30)\n",
    "        self.fc2 = nn.Linear(30, 10)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = x.view(x.shape[0], -1)\n",
    "        x = self.fc1(x)\n",
    "        x = F.logsigmoid(self.fc2(x))\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from mnist_loader import load_data_shared, vectorized_result\n",
    "training_data, validation_data, test_data = load_data_shared(filename=\"../mnist.pkl.gz\",\n",
    "                                                                     seed=666,\n",
    "                                                                     train_size=2000,\n",
    "                                                                     vali_size=0,\n",
    "                                                                     test_size=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(data, net):\n",
    "    with t.no_grad():\n",
    "        #for index in range(test_data[0].shape[0]):\n",
    "            # get the inputs\n",
    "        inputs, labels = t.Tensor(data[0]), data[1]\n",
    "\n",
    "        # forward + backward + optimize\n",
    "        outputs = net(inputs)\n",
    "        _, predicted = t.max(outputs, 1)\n",
    "\n",
    "        #print('Predicted: ', predicted)\n",
    "        #print('target: ', t.Tensor(labels).int())\n",
    "\n",
    "        correct = (predicted == t.Tensor(labels)).sum().item()\n",
    "        accuracy = correct / data[0].shape[0]\n",
    "        return accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fit(net, criterion, optimizer):\n",
    "    loss_scores = []\n",
    "    test_scores = []\n",
    "    train_scores = []\n",
    "    for epoch in range(100):  # loop over the dataset multiple times\n",
    "\n",
    "        # get the inputs\n",
    "        inputs, labels = t.Tensor(training_data[0]), t.Tensor(training_data[1])\n",
    "        vector_labels = t.Tensor([vectorized_result(y) for y in training_data[1]])\n",
    "        # zero the parameter gradients\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        # forward + backward + optimize\n",
    "        outputs = net(inputs)\n",
    "        loss = criterion(outputs, vector_labels.float())\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        # print statistics\n",
    "        loss_scores.append(loss.item())\n",
    "        train_scores.append(predict(training_data, net))\n",
    "        test_scores.append(predict(test_data, net))\n",
    "    print('Finished Training')\n",
    "    return loss_scores, train_scores, test_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Training\n"
     ]
    }
   ],
   "source": [
    "import torch.optim as optim\n",
    "net1 = Net()\n",
    "criterion = nn.MSELoss()\n",
    "optimizer = optim.SGD(net1.parameters(), lr = 1e-1)\n",
    "loss_scores, train_scores, test_scores = fit(net1, criterion, optimizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.plot(loss_scores)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3de3xU9Z3/8dcnd8iNBEIgCSHcr3INEQsq1dZLW6V2tUJrq9Uu2q61u7bd2u3+bGu3u9vL9qp1pd51W6XWWlQqtd4VhUy4CgEJGJKZBAgkIYSQy8x8f3+cCYQwISfJTCZz5vN8PPLIzDlnznyOg++c+Z7v+X7FGINSSinniot0AUoppcJLg14ppRxOg14ppRxOg14ppRxOg14ppRwuIdIFdDdq1ChTVFQU6TKUUiqqlJWVHTHG5ARbN+SCvqioCJfLFekylFIqqojIgZ7WadONUko5nK2gF5ErRGSPiFSIyF1B1l8kIptFxCsi13Zb9xMR2Ski5SLyaxGRUBWvlFKqd70GvYjEA/cBVwIzgZUiMrPbZlXATcDvu732I8ASYA4wG1gEXDzgqpVSStlmp42+BKgwxuwHEJGngOXArs4NjDGVgXX+bq81QAqQBAiQCBwacNVKKaVss9N0kw9Ud3nuDizrlTHmXeA1oDbws94YU959OxFZJSIuEXHV1dXZ2bVSSimb7AR9sDZ1WyOhichkYAZQgPXH4RIRueisnRmz2hhTbIwpzskJ2jtIKaVUP9kJejcwrsvzAqDG5v6vAd4zxjQbY5qBvwKL+1aiUkqpgbAT9KXAFBGZICJJwApgrc39VwEXi0iCiCRiXYg9q+lGKaViTYfPz/Pbali/8yA+f3iHi+/1YqwxxisitwPrgXjgYWPMThG5B3AZY9aKyCLgz0AWcJWI/MAYMwt4BrgE2IHV3POSMeb5cB2MUkoNttYOH21eqx9Kh8/P+55jlFbWs636GKMzkllUlM2ioixy0lMA8Pr8/GVrDQ++tZ+aY60ATBiVyq0XTeSaBfkkJ8SHvEYZahOPFBcXG70zVik11Pn8hgfe3McvX95Lu+/MDofxccK03HQONbVy9ER70NeXTMjmKxdPorXDx29f38cOzzFmjs3gxTuW0p/bjUSkzBhTHGzdkBsCQSmlhrrq+ha+sWYbmyrruXxWLiUTRgIQJzAtN515hSMYnpSAMYYPj5yg7EADTa3eU6+fN24EC8dnnXp+xewxvFNxlPqW9n6FfG806JVSKghjDJ7Gk+RlDiMuzgrf+hPtPPrOhzz8TiUA/3PdXD6zIL/HcBYRJuakMTEn7ZzvJSIsnTIqpPV3pUGvlFJB3PPCLh55p5LMYYkUj89iZFoSa7fV0Nrh57KZufy/T81kXPbwSJdpiwa9Ukp18/i7lTzyTiVXzc0jNSmeTZX1vLX3CFfPy+O2iycyeXR6pEvsEw16pZTq4vU9h/n+2p18bMZofnn9POIDzTbGmLC0nw8GDXqllMIK8tf2HOaOP2xlxtgMfrVi/qmQB6I25EGDXikV49q9fl7aeZD7X99HeW0T40cO56EbF5Ga7Jx4dM6RKKWUDcdbO9hc1Ujph/WUVtaztbqRNq+fSTmp/PTaOSyfl09SgrPmZNKgV0o5nt9veGRDJX8qc7P7YBN+Y93UNCsvg8+fP56lU0aybOroU90onUaDXinlaDWNJ7lzzVbe219P8fgs7rh0CouKspk3boSjmmfOJTaOUikVc9q9fp7d7OZH68rx+w0/uXYO1y0siOqLqv2lQa+UcpSWdi9Pbao+NWjYwvFZ/Pyzcxk/MjXSpUWMBr1SyjGqjrZww0MbqapvoaQomx995jyWTc2JybP4rjTolVKOsOfgcb7w0EbafX5+/+Xz+cjk8I0dE2006JVSUW9bdSM3PrKJpPg41tx6AVNzo2uIgnDToFdKRbWjzW3c+Mgm0lMS+L9bFlM4MjoGGhtMGvRKqaj245d209zqZc2tF2jI98BZt38ppWKKq7KeNS43t1w4QZtrzkGDXikVlbw+P//+3PvkZaZwxyVTIl3OkKZBr5SKSo+9e4DdB49z91WzYuYO1/6yFfQicoWI7BGRChG5K8j6i0Rks4h4ReTabusKReRvIlIuIrtEpCg0pSulnO5Em5d1O2qpPXby1DJjDI+/W8mPX9rNsmk5XD4rN3IFRole/wyKSDxwH/BxwA2UishaY8yuLptVATcB3wyyi8eBHxljXhaRNMAfZBullDql/kQ7j26o5LENlRw72UFivPDpefl8dtE47n21gjc+qGPZtBz+57q5MX8zlB12vu+UABXGmP0AIvIUsBw4FfTGmMrAujNCXERmAgnGmJcD2zWHpmyllBMZY3hyYxX/+WI5Jzt8XD4rl5Ulhby+p46nSqv4Y5mb5IQ4frh8FjcsHq8hb5OdoM8Hqrs8dwPn29z/VKBRRJ4FJgB/B+4yxvi6biQiq4BVAIWFhTZ3rZRykrrjbfzrM9t4bU8dF03N4e5PzTg1N+uyaaP52iWTeWnnQc6fMJLJo9MiXG10sRP0wf5kmj7s/0JgPlbzztNYTTwPnbEzY1YDqwGKi4vt7lsp5RAVh49z/QPv0dzm5QdXz+KLF5x9tj4yLZnPnz8+QhVGNztB7wbGdXleANTY3L8b2NKl2ec5YDHdgl4pFbuMMfzbn9/HZwwvfG0pU7Q/fMjZ6XVTCkwRkQkikgSsANba3H8pkCUiOYHnl9ClbV8ppf68xcOmD+v59hXTNeTDpNegN8Z4gduB9UA5sMYYs1NE7hGRqwFEZJGIuIHrgAdEZGfgtT6snjiviMgOrGag34XnUJRS0eZYSwf/ua6ceeNGcH3xuN5foPrF1l0Gxph1wLpuy+7u8rgUq0kn2GtfBuYMoEallEP97G97rK6UXypx7HytQ4HeGauUioj3Pcd4cuMBvnhBEbPzMyNdjqNp0CulIuI3r+4lPTmBOy+bGulSHE+DXik16PYeOs76nYe46SNFZKQkRrocx9OgV0oNuvvf2MewxHhuWjIh0qXEBA16pdSgcje0sHZrDStKxpGdmhTpcmKCBr1SalD97s39iMA/Xjgx0qXEDA16pdSgOdLcxlOl1VwzP5+8EcMiXU7M0KBXSg2KnTXHWLn6Pbx+w60XT4p0OTFFp2VRSoWV32948O39/HT9HkYMT+KRmxYxKUdHnxxMGvRKqbAxxnDXs9tZ43Jz+axc/uszc/QCbARo0Culwua3r+9jjcvN7R+dzDcum6oThUSIttErpcLixe21/HT9Hq6em6chH2Ea9EqpkNtW3cida7aycHwWP7l2joZ8hGnQK6VCqt3r5841WxmVlszqLywkJTE+0iXFPG2jV0qF1INv72df3QkeuWkRI9OSI12OQoNeKdUHPr/h95uqqDveRvH4LBaMzyIt+XSMuBta+PUre7l8Vi4fnT46gpWqrjTolVK2uBtauPPpbWyqrEcEjIE4gfMnjOTWiydy8dQc7nl+F4Jw91WzIl2u6kKDXinVq3U7avn2M9sxwP9cN5fLZuWytbqRTR/W80yZm5seKWXiqFT2HznBXVdOJ1+HNxhSxBgT6RrOUFxcbFwuV6TLUEoFeBpP8tGfvs6MvAzuXTmfcdnDz1jf7vXz3FYPD7yxj2FJ8Tz7lSUkJWg/j8EmImXGmOJg62x9GiJyhYjsEZEKEbkryPqLRGSziHhF5Nog6zNExCMi9/a9fKVUJP3y5Q9A4LefX3BWyAMkJcTx2eJxvPKNZTx/+1IN+SGo109EROKB+4ArgZnAShGZ2W2zKuAm4Pc97OaHwBv9L1MpFQl7Dx3nT5vdfHHxeFvNMdpffmiy86e3BKgwxuw3xrQDTwHLu25gjKk0xmwH/N1fLCILgVzgbyGoVyk1iH72tz0MT0rgqx+dHOlS1ADYCfp8oLrLc3dgWa9EJA74H+BbfS9NKRVJW6oaWL/zEKsumqgDkUU5O71ugn0Xs3sF96vAOmNM9bm+0onIKmAVQGFhoc1dK6VC6WhzG796ZS+Hm9oAKD/YxKi0JG5ZqvO6Rjs7Qe8GxnV5XgDU2Nz/BcCFIvJVIA1IEpFmY8wZF3SNMauB1WD1urG5b6VUiLy25zDf+uN2mk52MGFUKgDDkxL47idmkJqsvbCjnZ1PsBSYIiITAA+wAvicnZ0bYz7f+VhEbgKKu4e8UipyfH7DD1/YxaMbKpmWm84Tt5QwY2xGpMtSIdZrG70xxgvcDqwHyoE1xpidInKPiFwNICKLRMQNXAc8ICI7w1m0Uio0fvRiOY9uqORLS4r4y+1LNOQdSm+YUipGPfFuJf/vLzu5eckE7r6qe49pFW0GfMOUUspZXt9zmO8/v4tLp4/mu5+cEelyVJjpVRalYsixlg6eeK+S+1/fx7TcdH69cj7xcXqTk9Np0CvlIEea23i1/DBTctOYlZdJUkIcJ9t9bK1u5JXyQ/xhUxUn2n18dFoO//WZOdqjJkbop6yUQ7gbWrjhwY1UHm0BICUxjvHZqeyra8brN8THCZ+aM5bbLp6kF11jjAa9Ug5QcbiZLzy0kRNtXh750iJa232UVjawr66ZS2aMpqQomwWFWWQOT4x0qSoCNOiVinKbqxr4x8dciMBTqy5gZp51tn7leWMjXJkaKjTolYpSXp+fe1+r4DevVjA2M4XHby5hYk5apMtSQ5AGvVJRyN3Qwtf+sIUtVY1cMz+fHyyfRUaKNsuo4DTolYqw6voWnnjvAOnJCdyweDxZvYwU2XCinS88tIkjzW38euV8rp6bN0iVxrCmWlj/HfC29XMHAhd8FYqWnrl4y5OQnA4zlwd/WYho0CsVIeW1TfzvG/t4YXstAnj9hvvf2MfKkkJWlhQyKSf1rIk82r1+bn2yDE/jSf7wj+ezcHx2ZIqPNeVrYeefIXc29GdylSMVEJ9wZtAbA3//AaSN1qBXymk2fVjP/a9X8NqeOlKT4rll6QRuXjKBYyc7eOCNfTy6oZKH3v6QkalJFBdlsagom+KibGaOzeCuZ7ez6cN6frVinob8YHK7IH0sfOWd/r3+mZuhauOZy45Vw4nD0HIE2pohOXzXVzTolQqDlnYvLe0+ADp8fnZ6mig9UM+GiqPs8BwjOzWJb3x8Kl+8oOhUl8cxmSn8/Pp5fPPyabz5QR2llQ2UVtazfuchAJLi42j3+fmXj01l+Txbc/+oUPG4IH9h/1+fvxDe/xMcPwjpYwL7LLN+Gz/UboOiJQOvswca9EqF2JaqBm54cCMnAkHfKTFemJ2fyfevmsn1iwoZlhQf9PV5I4axoqSQFSXWJDyHmlpxBUI/Y1gid1yq0/oNqpZ6qN8P87/Q/33kB8Ya85TB9E9aj90uiEsAv9f6Q6JBr1R0aG7z8s9Pb2XE8CTuunKitVCEqaPTmDtuBCmJwcP9XHIzUvjknLF8co72i48Iz2br90DO6MfOsUK9a9B7NkPefGg+bIV+GGnQKxVC9zy/k+r6Fp5adQElE7QN3RE8ZYBYodxficMgd9bpQPd5oXYrLLjRaqfv3n4fYjpMsVIhsm5HLWtcbr66bLKGvJN4XJAzDVIGOD5Q/kKo2QJ+PxzeBR0t1rL8hdDkttrvw0TP6JXqpw0VR7j/jX20dfgB2FXbxNyCTL7+sSkRrkyFjDHWGf3UKwe+r/xicD0MR/eevhBbsBCa66zHXZt1QkzP6JXqozavjx+9uIvPPbiR/XUniI8T4uOECyaN5Ncr55MYr/9bOUZDJbQchfwFA99XZxu/22V9SxiWDVkTTrffh7GdXs/oVUx7fc9hXt51KOi6tOQE5hdmUVyURfbwJPYebqa0sp4n3zvA7oPHuWFxId/9xMwee88oBzh15h10hr6+GTUVktKtfXo2W8Evcrr9vvO9wkCDXsWsNaXV3PXsdlKTEkhOPPssvOmklwfe3A/A8KT4U/3i80cM46Ebi7l0Ru6g1qsiwFMGCSkwOgRz6sbFQf58+PBNOFoBM64+vS5/Iex4xmq/jwv9N0JbQS8iVwC/AuKBB40x/91t/UXAL4E5wApjzDOB5fOA+4EMwAf8yBjzdOjKV8qe7e5GahpbKS7KYlRaMg++tZ//eLGci6bm8MANC4Oelbd5fbzvacJVWY+n8SRzC0awqCibcdnDzhqaQDmU2wVj50F8iAaMyy+GD39uPe76LaFr+33OtNC8Vxe9Br2IxAP3AR8H3ECpiKw1xuzqslkVcBPwzW4vbwG+aIzZKyJ5QJmIrDfGNIakeqVseHazm289sx2f3wDWGbmn8SSfOG8Mv7x+PkkJwc+gkhPiWTg+i4XjswazXDVU+DqsO1YXfTl0++zaFz9vwdnL3a7IBD1QAlQYY/YDiMhTwHLgVNAbYyoD6/xdX2iM+aDL4xoROQzkABr0alA8tqGS763dyZLJI7njkilsrW7EdaCB5fPy+MZl03RibLsaq6y7Q3uTPQlGjAt/PeHibYPqTWB81jH72qyeMaHSeRafNQFSR55ePmoqJGdYF2nnfz507xdgJ+jzgeouz93A+X19IxEpAZKAfUHWrQJWARQWFvZ110qdobXDx7bqRv76/kEe3VDJx2fm8puV80lJjOf8iSO5NdIFRhtj4JFPwrGq3rcdORm+Fr6LimH33v3w9++dfi5xUFASuv2nj7H+GxUuPnN5XJx1Q1aYLsjaCfpgpzymL28iImOBJ4AbjTH+7uuNMauB1QDFxcV92rdSAMYYXt19mAff+hDXgXo6fNY/o88WF/Cf15xHgnZ57L+mGivkP/I1mPaJnrfb9RfY+L9w4gikjhq8+kKp6l3rbPvTv7WeD8sO/TeUm9dbF3i7u+yHkJga2vcKsBP0bqDrkRYANXbfQEQygBeBfzfGvNe38pQ6t5rGk7y99wgPvf0hew4dJ3/EMG5eOoGSomwWjs9ixPBzT+KhbPAE+nfPvObczRjGWEHvKYOplw9ObaHUeXPU5I/D+I+E7316+iM4dm7Y3tJO0JcCU0RkAuABVgCfs7NzEUkC/gw8boz5Y7+rVKqLQ02t/Gz9HjbsO4qn8SQAU3PT+MX1c/nUnDy9YSnUPGUQnwRjZp97u7x5VlOH2xWdQd9YBSfqQtsmP0T0GvTGGK+I3A6sx+pe+bAxZqeI3AO4jDFrRWQRVqBnAVeJyA+MMbOAzwIXASNF5KbALm8yxmwNx8Eo53vp/Vq+8+wOTnb4uHR6Ll++cAKLApNyxOmF1fBwl8GY8yAh+dzbJaVa/c3DeONPWHXWPZBRKocoW/3ojTHrgHXdlt3d5XEpVpNO99c9CTw5wBpVDDnc1MqfNnto9551KYeKumae31bDefmZ/HLFPCblhG9GHhXg91kDcc2/wd72+Quttnpj+jflXiR5yiA+2Zou0GH0zlg1ZKzfeZC7/rSdhpaOoOsT44V/+ugkvn7p1B77vqsQq9sNHSfsn+XmL4TNj1ldMUdOCm9toeYps9rJQ3Vz1BCiQa8i4khzG67KegL3MPHGnjqedlUzOz+DP952ARNHBT9b1+aZQdbXsV46t3O7oivofR1QsxWKvxTpSsJCg16FzIk2L1uqGnEdqCc1KYHPLhpH5rAzz46q61v43Vv7ebq0mrYuzTMi8NVlk/jnj+nZ+pDidkHKCMieaG/7nOlWF0GPC+ZeH97aQunwLvCedGT7PGjQqwEyxvDW3iM88OY+3ttfj89vELGaaH/1yl4+v7iQxRNGsqWqgU2V9ZRWNhAn8Jn5BVxfMo7UJOufYOawRMZkBulbrCLLU3Z6lEU74uLDeuNP2Dj4Qixo0Kt+MMZQXX+S9/Yf5fH3Knnf00RuRjK3XTyRkgkjWVA4gqr6Fv73jf387s39PPDGfuIEZuVlsuqiidx4QZGGejRoP2Gd6Z7rJqlg8hdY/em9bb331Bkq3GUwfCRkFUW6krDQoFe2Nba08+OXdvNK+WEOH28DYOKoVH7yD3NYPj+P5ITTI0DOysvkNyvn863LpuFubGFOwQjSkvWfW1Sp2QrG3/ex2AuKYUM7HHw/evqk9/WbS5TR//OULe9UHOEba7ZxpLmNT5w3lpIJ2SwqymbK6LRzXiAtHDmcwpHDB7FSFTL9bc7o3N7jio6gb22yehfNuibSlYSNBr06Q8OJdsoONLC1upGTHdZEG0eb23huaw0Tc1J57sYlzM7PjHCValB4XDBifN/HrcnIh7Qx1oXc86NgCLmaLYBxbPs8aNBHtTavj1+8vJe/lwefCq+r5IQ45o4bwaKiLOYWjCAl0WpmORkY6bE0cKG04nAzAPFxwrDANiLwxQvG850rZ+i0edGk7gP40y1WW3l/NFT2b7JqESs0y9fCvdv6996JKXDdo/Z7+3S1/ruw92X727ces36HYl7YIUqDPkp9cOg4d/xhC7sPHueiqTmk99L+3dTawfNba/j9xuBDzaanJLBwfBbXzM9nUVE2cwoyT/0xUFFqz4twcDvMXE7wQWh7kTsTzr+tf+99wT9BQpLV/aqvjA/Kn4cP1sPir/Tttb4OKH3Iuqjalwk8Rs+E4dl9e68ookEfZY6d7OCxDZXc+1oFGSkJPHxTMZdMtzd3qc9v2HPwOLtqm/D5rT7s8XFxzM7PYOrodL0ZyWncLuuM+LOPD/57Fy2xfvrr5zOt+vuqsz/8Rd+E867t//s7jAZ9FDDGUFXfwu83VvF/G6tobvNyxawx/Mc1sxmVZr/7WnycMDMvg5l5GWGsVg0Zns0DC9tIyl/Qv774Du8P318a9EOQz28or7UmpS490ICrsp5DTW3ECXxqTh63XTxJw1qdW1MNHK+xJp2ORvnFVvPNiaNnTrnXG4f3h+8vDfoIaff6T/Vq8fsN5QebcFU2UFpZz5aqRprbvADkZaaweOJIiouyWTY1h3HZ2lVR2dDXMWqGms66PWUw9TL7r/O4rD8SDu0P318a9BHw3v6jrHrcRVOr94zlIjAtN51Pz89jUVE2xUXZ5I8YFqEqVVRzuyAuMXqH3B0bmMSkL0Hf2gR1e2DWZ8JbWxTSoB9kr+4+xFee3My47OHccek4JHDmMXFUKgsKs8gc7rwhUlUEeAKThSRG6VATyWmQM+P0NIZ2dPaHj4abtAaZBn2IeX1+Ko+ewOs/u1vZ9upj/NufdzBjbAaP3VxCdqrOZ6rCoHOykLkrI13JwOQvgN0v2J/EpLO5Ks+5/eH7S4N+gDqH5i2trMd1wGpfb2n39bh9yYRsHrqxmPQUPXNXYVK3B9qbo7d9vlNBMWx5wv4kJp4yyJ7k6P7w/aVB3w8n232scVXzp81udtY04fMb4gSmj8nguoUFzB034tRdpV0lxsexdMoovRFJhZdTuhieGjNns/2gL7owvDVFKQ16mzr7sq/dWsMjGyqpP9HO3IJMvrpsEsVF2SwoHKFn6Wpo8LggJdM6u41mOTMgcbh1PHOuO/e2xzxwvDb6v8WEia2gF5ErgF8B8cCDxpj/7rb+IuCXwBxghTHmmS7rbgT+PfD0P4wxj4Wi8MGyYd8RnnzvAKWVDdQFhua9ZPpovrJsEouK9CuiGoI6h9yNi/KZuuITrElM7Nwh23nRNlrvGwizXoNeROKB+4CPA26gVETWGmN2ddmsCrgJ+Ga312YD3wOKAQOUBV7bEJryw6e1w8fP1u/hwbc/JCc9mSWTrL7sH5k0kok5weczVSri2lvg0C648M5IVxIa+Qtg4wPgbbfGzumJpwzik2BMlHYnDTM7Z/QlQIUxZj+AiDwFLAdOBb0xpjKwzt/ttZcDLxtj6gPrXwauAP4w4MrDaO+h43wtMGCYjtqowubABjhcHnzd+CUwenrf91m7zRoUzClntvnF4PsNvPUzSDvHmE57X7a6k0bLjFaDzE7Q5wPVXZ67gfNt7j/Ya/O7byQiq4BVAIWFhTZ3HR5lBxr40iObSEqI45GbFvHR6aMjWo9yKJ8X/u86q3dMMAWL4Mt/7/t+TzVhOKSLYeFiSEiBN37c+7YXfiP89UQpO0EfrAOr3bFHbb3WGLMaWA1QXFzcj3FNQ+PtvUdY9YSL0enJPPnl8ynI0uEGVJjU7bZC/pM/h+mfOnPdWz+Dskd7b64IxlMGIwohzSEnKOlj4FsVVpPUuYhAas7g1BSF7AS9GxjX5XkBUGNz/25gWbfXvm7ztYPq1d2HuO2JzUzMSeXxW0oYnR6ldxSq6NB55j1xGaR3a5IYvwQ2rYZDO/reRdJd5rw7Q5PTrR/Vb3Yuy5cCU0RkgogkASuAtTb3vx64TESyRCQLuCywbEhp8/r4zrM7mDQ6jadWLdaQV+HndsGwrOAzKHV2EXT3cZje5sNwrMo57fMqZHoNemOMF7gdK6DLgTXGmJ0ico+IXA0gIotExA1cBzwgIjsDr60Hfoj1x6IUuKfzwuxQ8uxmD4ea2vi3T0xnxHAdlkANAs9m62w92K39GfnWhce+jsfulBulVMjZ6kdvjFkHrOu27O4uj0uxmmWCvfZh4OEB1BhWPr/hgTf2Macgk6WT+zgJslL90dYMdeUw46rg60Wss/K+DOgF1rcEiYexcwdeo3KUKL+jYuDW7ail8mgLX1026dRIkkqFVe1WMP5zn3nnL4CjFXCyD7eceMqseV6TtBOBOlNMB70xht++vo9JOalcNnNMpMtRsaLzTs9zBf2piTc229un3x9oDtL2eXW2mA761/fUUV7bxG0XT9KJsdXg8ZRZU92da4q8vPmA2A/6+n3QdkzHelFBxWzQ+/2GX72yl7zMFJbPO+seLqXCx1PW+5l3SiaMmmq/nd7OtwQVs2I26P9YVs3W6kbuvGwaSQkx+59BDbamWmjy2Avk/IVWgBsb9xB6XJCUbv1xUKqbmEy4+hPt/Ndfd1NSlM0/LNCzeTWI+jJpd8FCaDkCjVX29ps3D+J0TCZ1tpgM+p+8tJvmVi8//PRs7WmjBpfHBXEJMGZO79t2Nu/01nzT0QoH39f2edWjmAv6sgMNPFVazc1LJzBtjN5WrQaZpwxyZ9ubtDt3ljWgV28XZA/uAH+Hts+rHsXcDFP3vLCLMRkpfP3SKZEuRfWm7gN46GO9D2gVTfwdsOjL9raNT7Rufnr3PmtM9p6YwOjgGvSqBzEV9B8cOs626ka+d9VMUpNj6tCj075XofUYXO3XbI0AABAgSURBVHC7NamEE0gczL/B/vaX/Qj2rOt9u6zxkJHX/7qUo8VU2j23xUN8nPCpOfo/RFTwuCA9Dy7/UaQriZxxi6wfpQYgZtrojTH8ZWsNSyaPIiddZ6GJCp4y50ygoVQExUzQlx1owNN4kk/P07P5qNBSD/X7tSeJUiEQM0H/3FYPKYlxXDZLx7SJCp09TfQCo1IDFhNB3+Hz8+L2Wj4+cwxpehE2OnhcgATGfFFKDURMBP2bH9TR0NKhzTbRxFMGOdN1CjmlQiAmgv65rTWMGJ7IhVN08uCoYIw1xovT5j5VKkIcH/RtXh+vlB/iytljdfCyaNHwIZys1/Z5pULE8cm3paqRlnYfH52mZ/NR49SFWO1xo1QoOD7o36k4QnycsHjSOSZ5UEOL2wUJw2D0zEhXopQj2Ap6EblCRPaISIWI3BVkfbKIPB1Yv1FEigLLE0XkMRHZISLlIvKd0Jbfu7crjjC3IJOMlMTBfmvVX51D7sZrDymlQqHXoBeReOA+4EpgJrBSRLqfat0CNBhjJgO/AH4cWH4dkGyMOQ9YCNza+UdgMDS1drCtupGlk0cN1luqgfK2Q+02bZ9XKoTsnDKVABXGmP0AIvIUsBzY1WWb5cD3A4+fAe4Va6B3A6SKSAIwDGgHmkJTejetx+DNn8L0q6DwfADe23cUv4ElGvQD43oYju4bnPdqbQRfmwa9UiFkJ+jzgeouz93A+T1tY4zxisgxYCRW6C8HaoHhwL8YY+q7v4GIrAJWARQWFvbxEAL8PtjwG8jIPxX0b1ccYVhiPPMLs/q3TwXNdfDCv1ijR8YNUvNXRj4ULR2c91IqBtgJ+mBTMHWfxLKnbUoAH5AHZAFvicjfO78dnNrQmNXAaoDi4mIbE2QG0XljTdvxU4verjjC+ROztVvlQHROffeF56BoSWRrUUr1i50EdAPjujwvAGp62ibQTJMJ1AOfA14yxnQYYw4D7wDh6TMXn2jNxtNmtQzVNJ5kf90JbZ8fKE+ZNYZ63rxIV6KU6ic7QV8KTBGRCSKSBKwA1nbbZi1wY+DxtcCrxhgDVAGXiCUVWAzsDk3pQSSnQ1szYHWrBFg6RYN+QDwuq5tjUmqkK1FK9VOvQW+M8QK3A+uBcmCNMWaniNwjIlcHNnsIGCkiFcCdQGcXzPuANOB9rD8Yjxhjtof4GE5LTj/VdPNOxRFGpSUxLVfHSuk3vz8wJrxeGFUqmtnqqGyMWQes67bs7i6PW7G6UnZ/XXOw5WGTlAZtxzHG8HbFUZZMHoXV+Uf1S/1+qzeTBr1SUc1ZVymTM6C9maMn2jnS3MbcghGRrii6eVzWb538Q6mo5rCgT4O2JqrqWwAYP3J4hAuKcp4ySEy1hgtWSkUthwW91UZfrUEfGm6XNfFHXHykK1FKDYADg76ZqqNW0BdkadD3m7cNDu7QMeGVcgBnBX3gYmxVfQu5GcmkJOqZaL8d3AH+Dr0Qq5QDOCvokzPA14bn6DEKs/VsfkA674jVMeGVinoOC3qrz/zR+nrGadAPjNsFaWMgQ+fZVSraOWvA7+Q0AFqON1CYbWPSCm87NB88/TwtFxKSe97eGGiqAeMbYKFRwF1qdavU+xCUinoOC3rrjD6VVntNN099DipePv18wsVwY/fRHbrY+AC89O0BFhlFFt4U6QqUUiHgyKBPo8Ve0B/eBYUfgfmfh90vQsUr1ll+QlLw7T98AzLHwbKzJtlynrgEmP7JSFehlAoBZwV9UiDo5WTvQe/3wfGDMHcFzL/BGrRrzzo49D7kLzh7e2OsdutJl1jbK6VUlHDkxdishDZy0s/R1g5wos5qa08faz3v7F3S2duku2NuOHFYhwNQSkUdhwW9dTG2MNXX+2BmTYEh9Tt7lWQWWBdj3a7g23eO+6L9ypVSUcZhQW+d0ecP8/a+bfegF7FCvKczek8ZxCdD7uwQFKqUUoPHUUFvEq3JMUYnd/S+8fFa63d6l37i+Qvh6F442XD29u4yGDun5wu1Sik1RDkq6I+e9HHCJDMqsb33jZtqrJ4lqTmnl3U2y9RsOXNbnxdqt2qzjVIqKjkq6KvqW2hmGFkJbb1vfLzWuvMzrst/gs7eNu5uzTd15dDRosMBKKWikqOCvrq+hWYzjAxp7X3jJs/Zt/enZMKoqWe3058a9yVIt0ullBriHBX0VUetM/rhpqX3jZtqIWPs2cvzi60eNsacXuZ2wbBsyJ4YumKVUmqQOCvo61toj08lvuNE7xsfrz3zQmyngoVWH/vGqtPLOifI1nFflFJRyFbQi8gVIrJHRCpE5Kz7/0UkWUSeDqzfKCJFXdbNEZF3RWSniOwQkZTQlX+mqvoWTGBM+nNqbYL25uAjM3ZecO1srmk7DofL9UKsUipq9Rr0IhIP3AdcCcwEVopI96EhbwEajDGTgV8APw68NgF4ErjNGDMLWAbY6PvYP9X1LUhKeu9B370PfVe5s63+8p1BX7MVMHpHrFIqatkZ66YEqDDG7AcQkaeA5cCuLtssB74fePwMcK9Yt6ZeBmw3xmwDMMYcDVHdZ2nz+qhtaiUxOxMaewn644GgTw/SRh+fCGPnwvY1UL8fGqut5Xl6IVYpFZ3sNN3kA9VdnrsDy4JuY4zxAseAkcBUwIjIehHZLCL/GuwNRGSViLhExFVXV9fXYwDgWEsH5+VnkpY+wjqj73oxtbumwM1SwS7GAhTfDOm5cKwaBGsQs9SR/apLKaUizc4ZfbArkN1TtKdtEoClwCKgBXhFRMqMMa+csaExq4HVAMXFxedI6J6Nzkhh7e1L4a2N8IEXvK2QOCz4xqfO6HuYPWneSutHKaUcwM4ZvRsY1+V5AVDT0zaBdvlMoD6w/A1jzBFjTAuwDghvG0hyhvW7rbnnbZpqrO6SiWG7LqyUUkOGnaAvBaaIyAQRSQJWAN2nYVoL3Bh4fC3wqjHGAOuBOSIyPPAH4GLObNsPvSRrBEvamnrepqlW50JVSsWMXptujDFeEbkdK7TjgYeNMTtF5B7AZYxZCzwEPCEiFVhn8isCr20QkZ9j/bEwwDpjzIthOhZLYARL2s9xRn+8RoNeKRUzbM0wZYxZh9Xs0nXZ3V0etwLX9fDaJ7G6WA6OzqA/VxfLploYO29w6lFKqQhz1J2xwKnJR3oMem+7NVOUntErpWKEA4O+l4uxzQet38H60CullAM5L+h7uxh7qg9991sBlFLKmZwX9L210Xf2oe/pZimllHIY5wV9UiogPfe6aTrH8AdKKeVAzgt6Eeusvqcz+qYaSEiBYVmDW5dSSkWI84Iezh30xwM3S+nY8kqpGGGrH33U6R709ftPTyRS90HPY9wopZQDOTPou04+4vfB7y6Bkw2n18//QmTqUkqpCHBm0Cenn74Ye+QDK+Qv/jZMXGYtG3NepCpTSqlB59ygPx7oL+92Wb9nXws5UyNXk1JKRYjzL8Z6yiA5E0ZOjmxNSikVIQ4O+kDTjccF+fMhzpmHqpRSvXFm+iWlWUMgtJ+AQ7sgXyf2VkrFLmcGfXI6YKDqXTA+yF8Y6YqUUipiHBz0wP7Xrd8a9EqpGObsoN/3OmSOg/TciJajlFKR5OygP7RDz+aVUjHP2UEPUKAXYpVSsc2ZQd85+QjoGb1SKubZCnoRuUJE9ohIhYjcFWR9sog8HVi/UUSKuq0vFJFmEflmaMruRecZvcTD2LmD8pZKKTVU9Rr0IhIP3AdcCcwEVorIzG6b3QI0GGMmA78Aftxt/S+Avw68XJs6g370zMBEJEopFbvsnNGXABXGmP3GmHbgKWB5t22WA48FHj8DXCpiDfguIp8G9gM7Q1OyDZ1BX6DNNkopZSfo84HqLs/dgWVBtzHGeIFjwEgRSQW+DfzgXG8gIqtExCUirrq6Oru19yxxGFz6PSi5deD7UkqpKGcn6INNxWRsbvMD4BfGmB4mcA1saMxqY0yxMaY4JyfHRkk2XHgn5HZvYVJKqdhjZ5hiNzCuy/MCoKaHbdwikgBkAvXA+cC1IvITYATgF5FWY8y9A65cKaWULXaCvhSYIiITAA+wAvhct23WAjcC7wLXAq8aYwxwYecGIvJ9oFlDXimlBlevQW+M8YrI7cB6IB542BizU0TuAVzGmLXAQ8ATIlKBdSa/IpxFK6WUsk+sE++ho7i42LhcrkiXoZRSUUVEyowxQYcCcOadsUoppU7RoFdKKYfToFdKKYfToFdKKYcbchdjRaQOODCAXYwCjoSonGgRi8cMsXncsXjMEJvH3ddjHm+MCXrH6ZAL+oESEVdPV56dKhaPGWLzuGPxmCE2jzuUx6xNN0op5XAa9Eop5XBODPrVkS4gAmLxmCE2jzsWjxli87hDdsyOa6NXSil1Jiee0SullOpCg14ppRzOMUHf2wTmTiEi40TkNREpF5GdIvL1wPJsEXlZRPYGfmdFutZQE5F4EdkiIi8Enk8ITEa/NzA5fVKkaww1ERkhIs+IyO7AZ36B0z9rEfmXwL/t90XkDyKS4sTPWkQeFpHDIvJ+l2VBP1ux/DqQb9tFZEFf3ssRQW9zAnOn8ALfMMbMABYD/xQ41ruAV4wxU4BXAs+d5utAeZfnP8aawWwK0IA1Sb3T/Ap4yRgzHZiLdfyO/axFJB+4Ayg2xszGGhp9Bc78rB8Frui2rKfP9kpgSuBnFXB/X97IEUGPvQnMHcEYU2uM2Rx4fBzrf/x8zpyg/THg05GpMDxEpAD4JPBg4LkAl2BNRg/OPOYM4CKs+R4wxrQbYxpx+GeNNU/GsMBsdcOBWhz4WRtj3sSav6Ornj7b5cDjxvIeMEJExtp9L6cEvZ0JzB1HRIqA+cBGINcYUwvWHwNgdOQqC4tfAv8K+APPRwKNgcnowZmf+USgDngk0GT1oIik4uDP2hjjAX4GVGEF/DGgDOd/1p16+mwHlHFOCXo7E5g7ioikAX8C/tkY0xTpesJJRD4FHDbGlHVdHGRTp33mCcAC4H5jzHzgBA5qpgkm0Ca9HJgA5AGpWM0W3Tnts+7NgP69OyXo7Uxg7hgikogV8v9njHk2sPhQ51e5wO/DkaovDJYAV4tIJVaz3CVYZ/gjAl/vwZmfuRtwG2M2Bp4/gxX8Tv6sPwZ8aIypM8Z0AM8CH8H5n3Wnnj7bAWWcU4L+1ATmgavxK7AmLHecQNv0Q0C5MebnXVZ1TtBO4PdfBru2cDHGfMcYU2CMKcL6bF81xnweeA1rMnpw2DEDGGMOAtUiMi2w6FJgFw7+rLGabBaLyPDAv/XOY3b0Z91FT5/tWuCLgd43i4FjnU08thhjHPEDfAL4ANgHfDfS9YTxOJdifWXbDmwN/HwCq836FWBv4Hd2pGsN0/EvA14IPJ4IbAIqgD8CyZGuLwzHOw9wBT7v54Asp3/WwA+A3cD7wBNAshM/a+APWNchOrDO2G/p6bPFarq5L5BvO7B6Jdl+Lx0CQSmlHM4pTTdKKaV6oEGvlFIOp0GvlFIOp0GvlFIOp0GvlFIOp0GvlFIOp0GvlFIO9/8BkbYrW00tPd8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.plot(train_scores)\n",
    "plt.plot(test_scores)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sigmoid Unit + Cross Entropy Cost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fit2(net, criterion, optimizer):\n",
    "    loss_scores = []\n",
    "    test_scores = []\n",
    "    train_scores = []\n",
    "    for epoch in range(100):  # loop over the dataset multiple times\n",
    "\n",
    "        # get the inputs\n",
    "        inputs, labels = t.Tensor(training_data[0]), t.Tensor(training_data[1])\n",
    "        vector_labels = t.Tensor([vectorized_result(y) for y in training_data[1]])\n",
    "        # zero the parameter gradients\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        # forward + backward + optimize\n",
    "        outputs = net(inputs)\n",
    "        loss = criterion(outputs, labels.long())\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        # print statistics\n",
    "        loss_scores.append(loss.item())\n",
    "        train_scores.append(predict(training_data, net))\n",
    "        test_scores.append(predict(test_data, net))\n",
    "    print('Finished Training')\n",
    "    return loss_scores, train_scores, test_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Training\n"
     ]
    }
   ],
   "source": [
    "import torch.optim as optim\n",
    "net2 = Net()\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(net2.parameters(), lr = 1e-1)\n",
    "loss_scores2, train_scores2, test_scores2 = fit2(net2, criterion, optimizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAeKUlEQVR4nO3de3hc9Z3f8fd3ZnS/W5Kvki3ZCIwwvoAgEBKaJU5rcrHThhDyJE1o05JNQ0I2293Csy1Pl3SfZxO2ubVsGpYkm02zIYRmEy8hccJtuSwGi2BsfBc2tuWbZNmWLOs6M9/+MSMxNpI1tiWP5szn9TzzzJxzfjPne55jf+an3zlzjrk7IiKS/UKZLkBERCaHAl1EJCAU6CIiAaFAFxEJCAW6iEhARDK14pqaGm9oaMjU6kVEstIrr7xy1N1rx1qWsUBvaGigtbU1U6sXEclKZrZ3vGUachERCQgFuohIQCjQRUQCQoEuIhIQCnQRkYBQoIuIBIQCXUQkILIu0FvfPMZXf7MdXfZXROR0WRfom9q7+c4zb9B1aijTpYiITCtZF+iNtSUAvHn0VIYrERGZXrIv0KsTgb5bgS4icpqsC/S6qiIiIVMPXUTkDFkX6JFwiPkzinmzS4EuIpIq6wIdoKGmhN2dCnQRkVTZGejVJezt6tOpiyIiKbIy0BtrS+gfjnGkZzDTpYiITBvZGeijZ7r0ZrgSEZHpIysDvaGmGIA3j/ZluBIRkekjKwN9bkUR+ZGQznQREUmRlYEeChkN1cU600VEJEVWBjokznRRD11E5C1ZG+iNtSXs6+ojFtepiyIikM2BXl3CUCzOwRP9mS5FRGRayNpAb6hJnLq4R9d0EREBsjjQG5OBrnF0EZGErA30mWUFFOeH1UMXEUnK2kA3MxqqSxToIiJJWRvokBh20XXRRUQS0gp0M1tlZjvMrM3M7h6nza1mttXMtpjZ309umWNrrClh//F+hmPxi7E6EZFpLTJRAzMLAw8A7wPagQ1mttbdt6a0aQLuAW5w9+NmNnOqCk7VWFNCLO7s7erjkpmlF2OVIiLTVjo99GuBNnff7e5DwMPAmjPa/EfgAXc/DuDuHZNb5tgum10GwI7DJy/G6kREprV0An0esD9luj05L9WlwKVm9oKZrTezVZNV4NlcMrOUcMjYfrjnYqxORGRam3DIBbAx5p35e/sI0AS8B6gDnjOzJe5+4rQPMrsDuANg/vz551zsmQrzwjTWlLBdPXQRkbR66O1Afcp0HXBwjDa/dPdhd98D7CAR8Kdx9wfdvcXdW2pra8+35tNcNrtMPXQREdIL9A1Ak5k1mlk+cBuw9ow2vwD+AMDMakgMweyezELHc/nsMvYf66d3MHoxViciMm1NGOjuHgXuBNYB24BH3H2Lmd1nZquTzdYBXWa2FXga+BN375qqolNdNrsc0IFREZF0xtBx98eBx8+Yd2/Kawe+nHxcVItTznS5ekHVxV69iMi0kdW/FAWoqyqitCCicXQRyXlZH+hmxqWzSnWmi4jkvKwPdIDFc8rZfqiHxMiPiEhuCkagzy6jZyDK4Z6BTJciIpIxAQn0xJku2w9p2EVEclcgAv2yWYkzXTSOLiK5LBCBXlGcx9yKQp3pIiI5LRCBDolLAOjHRSKSywIT6IvnlNPW0ctQVDe7EJHcFJxAn11GNO60dfRmuhQRkYwITKBfMbcCgNcPdme4EhGRzAhMoC+sKaEkP8zrBxToIpKbAhPooZBxxdwKNivQRSRHBSbQAa6sq2DrwR6iMR0YFZHcE6xAn1fBYDTOLh0YFZEcFKxAr0scGNWwi4jkokAFemN1CaUFETa3K9BFJPcEKtBDIaN5brl66CKSkwIV6ABL51Ww7VAPwzowKiI5JnCBfmVd8sDoER0YFZHcErxAn5f8xaiGXUQkxwQu0BtGDowq0EUkxwQu0BO/GC1nkwJdRHJM4AIdYGmdDoyKSO4JZKAvmVfBUDTOziO64YWI5I5ABvqyukoAXtuvYRcRyR2BDPQF1cVUFeexcf/xTJciInLRpBXoZrbKzHaYWZuZ3T3G8tvNrNPMNiYf/2HyS02fmbFifhWv7juRyTJERC6qCQPdzMLAA8DNQDPwcTNrHqPpT919efLx0CTXec5W1Feyq6OX7v7hTJciInJRpNNDvxZoc/fd7j4EPAysmdqyLtyK+VUAbGpXL11EckM6gT4P2J8y3Z6cd6aPmNkmM3vUzOrH+iAzu8PMWs2stbOz8zzKTd/S+grM0LCLiOSMdALdxpjnZ0z/I9Dg7kuBJ4AfjvVB7v6gu7e4e0ttbe25VXqOygvzuKS2lI37FegikhvSCfR2ILXHXQccTG3g7l3uPpic/Bvg6skp78KsmF/Jq/uO437m94+ISPCkE+gbgCYzazSzfOA2YG1qAzObkzK5Gtg2eSWevxXzqzjeN8zerr5MlyIiMuUiEzVw96iZ3QmsA8LA9919i5ndB7S6+1rgi2a2GogCx4Dbp7DmtK2Yn/iB0av7j9NQU5LhakREptaEgQ7g7o8Dj58x796U1/cA90xuaReuaWYZJflhXt13gn+9oi7T5YiITKlA/lJ0RDhkLKuv1IFREckJgQ50gOX1lWw92MPAcCzTpYiITKnAB/qK+VVE464bXohI4OVAoCcOjP5+ry7UJSLBFvhAryktYGFNCRveVKCLSLAFPtABWhqqeGXvMeJx/cBIRIIrRwJ9Bsf7htl9tDfTpYiITJmcCPRrGmYAaNhFRAItJwK9obqYmtJ8Nrx5LNOliIhMmZwIdDOjZcEMWtVDF5EAy4lAh8SB0X3H+jjSM5DpUkREpkTOBPrIOLp66SISVDkT6M1zyynKC2scXUQCK2cCPS8cYsX8Slr3KtBFJJhyJtAhcT761oM9nBwYznQpIiKTLqcC/ZqGKuKuG0eLSDDlVKCvmF9FOGS8vEfDLiISPDkV6KUFEa6cV8GLu7syXYqIyKTLqUAHuH5RNa/tP0HfUDTTpYiITKqcC/TrFlYTjbvORxeRwMm5QG9ZUEUkZBp2EZHAyblALymIsKy+kvUKdBEJmJwLdIDrFs5gU3s3vYMaRxeR4MjJQL9+YQ2xuNOqywCISIDkZKBfvaCKvLDG0UUkWHIy0Ivywyyvr2T9Gwp0EQmOnAx0gOsXVrP5QLeu6yIigZFWoJvZKjPbYWZtZnb3WdrdYmZuZi2TV+LUuG5RNXFHl9MVkcCYMNDNLAw8ANwMNAMfN7PmMdqVAV8EXprsIqfCVfOryI+EeKFNwy4iEgzp9NCvBdrcfbe7DwEPA2vGaPcV4GtAVtzjrTAvzLUNM3huV2emSxERmRTpBPo8YH/KdHty3igzWwHUu/tjZ/sgM7vDzFrNrLWzM/NBeuOlNew80svh7qz4DhIROat0At3GmOejC81CwDeAP57og9z9QXdvcfeW2tra9KucIu9uStSgXrqIBEE6gd4O1KdM1wEHU6bLgCXAM2b2JnAdsDYbDowunl1GbVkBz+46mulSREQuWDqBvgFoMrNGM8sHbgPWjix09253r3H3BndvANYDq929dUoqnkRmxrubanh+VyfxuE/8BhGRaWzCQHf3KHAnsA7YBjzi7lvM7D4zWz3VBU61G5tqOd43zOsHuzNdiojIBYmk08jdHwceP2PeveO0fc+Fl3XxvKupBoDndh1laV1lhqsRETl/OftL0RE1pQVcMbecZ3fqwKiIZLecD3RInO3yyt7jupyuiGQ1BTqJ89GjcdfFukQkqynQSVxOtzg/zDM7OzJdiojIeVOgAwWRMO+6pIantnXgrtMXRSQ7KdCTVjbP4mD3AFsP9WS6FBGR86JAT7pp8UzM4ImtGnYRkeykQE+qKS1gRX0lT2w7kulSRETOiwI9xcrmWWw+0K2rL4pIVlKgp3jf5bMAeHK7eukikn0U6CkumVnK/BnFPLFVgS4i2UeBnsLMWHn5LF54o4u+If1qVESyiwL9DCubZzIUjfOcrpEuIllGgX6GaxpmUF4Y4bdbNOwiItlFgX6GvHCIlc2z+N3WwwxF45kuR0QkbQr0MXxo6Vx6BqK616iIZBUF+hhuuKSGiqI8Htt0KNOliIikTYE+hvxIiFVXzOZ3W48wMBzLdDkiImlRoI/jg8vm0DsY5ZkdGnYRkeygQB/H9QurqS7J57FNBzNdiohIWhTo44iEQ6xaMpsnt3XoR0YikhUU6GfxgaVz6B+O8fR2DbuIyPSnQD+LdzRWU1tWwNrXDmS6FBGRCSnQzyIcMtYsm8tT2zvo6h3MdDkiImelQJ/ArdfUMxxz/uFV9dJFZHpToE/g0lllLK+v5JHW/bqBtIhMa2kFupmtMrMdZtZmZnePsfwPzWyzmW00s+fNrHnyS82cW1vq2Xmkl03t3ZkuRURkXBMGupmFgQeAm4Fm4ONjBPbfu/uV7r4c+Brw9UmvNIM+uGwOhXkhHmndn+lSRETGlU4P/Vqgzd13u/sQ8DCwJrWBu/ekTJYAgRqbKC/M4/1L5rB240H6h3QpABGZntIJ9HlAate0PTnvNGb2eTN7g0QP/YtjfZCZ3WFmrWbW2tmZXed233pNPScHo/xmiy7YJSLTUzqBbmPMe1sP3N0fcPdFwH8B/utYH+TuD7p7i7u31NbWnlulGfaOxhksqC7mJy9r2EVEpqd0Ar0dqE+ZrgPOdoGTh4EPX0hR05GZ8Yl3zOflPcfYerBn4jeIiFxk6QT6BqDJzBrNLB+4DVib2sDMmlImPwDsmrwSp4+PtcynOD/MD17Yk+lSRETeZsJAd/cocCewDtgGPOLuW8zsPjNbnWx2p5ltMbONwJeBT09ZxRlUUZzHLVfX8cuNBzmqX46KyDQTSaeRuz8OPH7GvHtTXt81yXVNW7e/s4G/e3EvP16/j7tWNk38BhGRi0S/FD1HC2tLuWnxTH60fi+DUZ3CKCLThwL9PPz7Gxo52jvIY6/pFEYRmT4U6OfhhkuquXRWKQ89v0fXdxGRaUOBfh7MjM/euIhth3r43dYjmS5HRARQoJ+3Ncvn0lBdzDef2KVeuohMCwr08xQJh/jCTU1sPdTDb9VLF5FpQIF+AdYsn0tjTQnfUi9dRKYBBfoFSPTSL2HroR7WbVEvXUQyS4F+gVYvS/TSv/nETmJx9dJFJHMU6BcoEg7xR++7lO2HT/Iz3QBDRDJIgT4JPrR0Di0Lqvir3+7g5MBwpssRkRylQJ8EZsa9H2rmaO8Q//uptkyXIyI5SoE+SZbWVfLRq+v4/gt72HP0VKbLEZEcpECfRH+y6jLywyH+4ldbM12KiOQgBfokmllWyBfe28QT2zp4fLMu3CUiF5cCfZJ95l2NLJlXzn/7xescOzWU6XJEJIco0CdZXjjE/bcso2dgmP++dkumyxGRHKJAnwKXzynnCzc1sfa1g6zbcjjT5YhIjlCgT5HPvWcRzXPK+bN/eF33HxWRi0KBPkXywiG+/rFlnBwY5ksPb9RlAURkyinQp9Di2eXct+YKnm87yref3JXpckQk4BToU+zWlno+clUd335qF8/u7Mx0OSISYAr0KWZm/I8PL+HSmWV86acb2X+sL9MliUhAKdAvgqL8MH/9yauIxuL8u7/dQHefLuAlIpNPgX6RLKot5cFPtbC36xR3/KiVwWgs0yWJSMAo0C+i6xZWc/8ty3hpzzH+9NFNxHXmi4hMokimC8g1H14xjwMn+rl/3Q7KCiN8Zc0SzCzTZYlIAKTVQzezVWa2w8zazOzuMZZ/2cy2mtkmM3vSzBZMfqnB8Z/es4jP3riQ/7t+H/f+cotuMC0ik2LCHrqZhYEHgPcB7cAGM1vr7qnXiH0VaHH3PjP7HPA14GNTUXAQmBl337wYBx58djdm8Oerr1BPXUQuSDpDLtcCbe6+G8DMHgbWAKOB7u5Pp7RfD3xyMosMIjPjnpsXA4lQ7x2M8pf/Zin5ER3WEJHzk06gzwNS737cDrzjLO0/A/x6rAVmdgdwB8D8+fPTLDG4RkK9JD/CN57YSefJQf76E1dRVpiX6dJEJAul0x0caxxgzEFfM/sk0ALcP9Zyd3/Q3VvcvaW2tjb9KgPMzLhrZRP337KUF9/o4tbvrufgif5MlyUiWSidQG8H6lOm64CDZzYys5XAnwGr3V2XFzxHH22p5/u3X8P+Y3188H89zwttRzNdkohkmXQCfQPQZGaNZpYP3AasTW1gZiuA75II847JLzM33HhpLb+88waqS/L5t997iQeebtO56iKStgkD3d2jwJ3AOmAb8Ii7bzGz+8xsdbLZ/UAp8DMz22hma8f5OJnAotpSfvH5G/jA0rncv24Hn/7ByxzuHsh0WSKSBSxT50C3tLR4a2trRtadDdydH7+0j7/41TbywsZXPryE1cvm6tRGkRxnZq+4e8tYy3SO3DRlZnzyugX8+q53c8nMUu56eCN3/OgVDuiAqYiMQ4E+zTXUlPCzP3wn99y8mOd3HWXl//wnvvPMGwxF45kuTUSmGQV6FgiHjM/+i0X87ss38q6mGr76m+38q28+y683H9JlA0RklAI9i9RVFfM3n2rhB7dfQyRkfO7Hv+cj3/ln1u/uynRpIjINKNCz0B8snsmv73o3X/3IlRw40c9tD67n1u++yAttR9VjF8lhOsslyw0Mx/jJy/v4P//0Bkd6BllWX8ln3tXIzUtmkxfW97VI0JztLBcFekAMDMd49JV2HnpuN2929TG3opBPXr+Aj15dT21ZQabLE5FJokDPIfG489T2Dr73/B5e3N1FXtj4l82z+dg19dxwSQ3hkM5jF8lmZwt03bEoYEIhY2XzLFY2z6Kt4yQPv7yfR3/fzq82H2JmWQFrls9lzfJ5XDG3XD9SEgkY9dBzwMBwjKe3d/DzVw/wzI4OhmPOgupibl4yh5uXzObKeRWE1HMXyQoacpFRx08NsW7LYX61+RD//EYXsbgzs6yA914+k5sWz+Kdi6opKdAfbiLTlQJdxnT81BBPbu/gyW1HeHZnJ6eGYuSFjasXVPHuplreuaiaK+dVENHZMiLThgJdJjQYjbFhz3Gea+vk2Z1H2XaoB4CS/DDXNM7gmoYZtCyoYll9JYV54QxXK5K7FOhyzo72DvLS7mO8uPso63cfo62jF4C8sNE8p5zl9ZUsn1/JlfMqWVhTojF4kYtEgS4X7PipIV7Ze5wNe4+xcd8JNh/opm8oBiR68VfMraB5bjnNc8ppnlvOJTNL1ZMXmQI6bVEuWFVJ/ujpkADRWJxdHb1sPtDN6we62Xygm59u2E//cCLkQ5a4UuRls8pomlnKopmlLKpNPIryFfQiU0GBLuclEg5x+ZxyLp9Tzq0tiVvOxuLO3q5TbDt0kh1HTrLz8El2HD7Jui2HSb2T3tyKQhprS2isKaGhuoQF1SUsqC6mvqpYYS9yARToMmnCIWNhbSkLa0v5AHNG5w9GY7x5tI+2jl52d/ay++gpdnf28o+vHaK7f/i0z6gpLaB+RhF1VcXMqyxiXlUR8yoLmVtZxJyKIsoLI/pBlMg4FOgy5QoiYS6bXcZls8vetuxE3xB7u/p4s+sU7cf72X+sj33H+tjUfoLfvH6I4djpx3hK8sPMrihkdkUhs8qTj7ICZpYXMqu8gJllhdSWFWj8XnKSAl0yqrI4n8rifJbVV75tWTzudJwc5GB3PwdPJB6Huwc53NPPoe4BXtp9jCM9A0Tjbz+wX1YQoaasgJrSfGpKC6guzWdGSWJ6RknKI7n+/IjOtZfsp0CXaSsUstHe+FXzq8ZsE487x/qG6OgZpOPkAB0nBznaO0jnycSjq3eIto5e1u8e5ET/MOOd1FWSH6aqJJ+q4nwqi/OoKMqjsjiPyqJ8KooS0+VFeZQXRRKvCxPTZQURnbIp04YCXbJaKGTUlBZQU1pAM+VnbRuNxTnRP0xX7xDHTg1xvG+IrlNDnDg1xPG+YY73DXGib4gT/cO0H++nu3+Y7v5hYmP8BZCqtCBCWeHII4+ywkhyXuJ1SX6E0sIIpQVhSgoilBQklhfnh5PPienCvJCOD8gFUaBLzoiEQ6Phny53p3cwyom+YXoGEgHf0z9Mz0B09PnkwDAnk9OnhqIcO5U4LtA7GKV3IDp6KudEzKA4L0xxMuyL80eewxTlJZ+T84rywhSlPBfmhSjKC1OYfBSd9hyiIPmcH9aXRpAp0EXOwsySPe288/6MaCzOqaEYvYNRTg1GR58TjxinhhLz+odi9A3FODUYpS/5ui+5rPPk4Oi8geHE/An+cBhne6AgEkoEfyRMQV5o9LkgEqIgEh5dXhAJkZ98FIw+h8eczg+HyI8Y+eHw6HtS5+VFjPxwiLyR+eGQhqqmgAJdZIpFwiEqikJUFJ3/l8KZ3J2hWJyBoTj9yYAfGE68Hhh9nD49GI2ftmww+ta8oWicgeE4p4aiHO+LnzZ/KBZnMNn+fL5ExhMOGXlhIy8Z8HnhEHmRxHReKPE6EkosiyTbjbSPjLwOnb4sEg6RF0o8R1KWR0bmhRJtR9YdDr3V7q15ifWOvG+86fDI5ybnTYe/fBToIlnIzJK96TAVTN4XxUSisUTYjwT9UPSt6cFojOGYJ5fFGIr6aJvhWOIx8r7oaLvTl0VjPjov9fVQNPFXTqJNnGg88f5o/K120ZgTjcffdqrrxRIeDXsjbDYa+CFLzgsn54eML628lA8tmzvpNaQV6Ga2CvgWEAYecve/PGP5jcA3gaXAbe7+6GQXKiKZF0n2jkum8W1q3Z1Y3InGffSLIRr30fCPxp1oLBH8sbgzHI8nnmOJ55E2seSXQ9yd4VjiPbHkZyfeG09+VmLeyGfF4yPvf+sRjftb892pLJ6aL+EJA93MwsADwPuAdmCDma11960pzfYBtwP/eSqKFBFJl5klhkfC5NwPzNLpoV8LtLn7bgAzexhYA4wGuru/mVwWn4IaRUQkDen8PG4esD9luj05T0REppF0An2sQ7fnddTBzO4ws1Yza+3s7DyfjxARkXGkE+jtQH3KdB1w8HxW5u4PunuLu7fU1taez0eIiMg40gn0DUCTmTWaWT5wG7B2assSEZFzNWGgu3sUuBNYB2wDHnH3LWZ2n5mtBjCza8ysHfgo8F0z2zKVRYuIyNuldR66uz8OPH7GvHtTXm8gMRQjIiIZootAi4gEhPl4F4ie6hWbdQJ7z/PtNcDRSSwnW+TidufiNkNubncubjOc+3YvcPcxzyrJWKBfCDNrdfeWTNdxseXidufiNkNubncubjNM7nZryEVEJCAU6CIiAZGtgf5gpgvIkFzc7lzcZsjN7c7FbYZJ3O6sHEMXEZG3y9YeuoiInEGBLiISEFkX6Ga2ysx2mFmbmd2d6XqmgpnVm9nTZrbNzLaY2V3J+TPM7Hdmtiv5XJXpWiebmYXN7FUzeyw53WhmLyW3+afJ6wkFiplVmtmjZrY9uc+vz5F9/UfJf9+vm9lPzKwwaPvbzL5vZh1m9nrKvDH3rSV8O5ltm8zsqnNdX1YFesrdk24GmoGPm1lzZquaElHgj939cuA64PPJ7bwbeNLdm4Ank9NBcxeJawaN+CrwjeQ2Hwc+k5Gqpta3gN+4+2JgGYntD/S+NrN5wBeBFndfQuL2lrcRvP39t8CqM+aNt29vBpqSjzuA75zryrIq0Em5e5K7DwEjd08KFHc/5O6/T74+SeI/+DwS2/rDZLMfAh/OTIVTw8zqgA8ADyWnDbgJGLlHbRC3uRy4EfgegLsPufsJAr6vkyJAkZlFgGLgEAHb3+7+LHDsjNnj7ds1wN95wnqg0szmnMv6si3Qc+7uSWbWAKwAXgJmufshSIQ+MDNzlU2JbwJ/CozcyrAaOJG84icEc38vBDqBHySHmh4ysxICvq/d/QDwVyTuR3wI6AZeIfj7G8bftxecb9kW6JN296RsYGalwP8DvuTuPZmuZyqZ2QeBDnd/JXX2GE2Dtr8jwFXAd9x9BXCKgA2vjCU5brwGaATmAiUkhhzOFLT9fTYX/O892wJ90u6eNN2ZWR6JMP+xu/88OfvIyJ9gyeeOTNU3BW4AVpvZmySG0m4i0WOvTP5JDsHc3+1Au7u/lJx+lETAB3lfA6wE9rh7p7sPAz8H3knw9zeMv28vON+yLdBz4u5JybHj7wHb3P3rKYvWAp9Ovv408MuLXdtUcfd73L3O3RtI7Nen3P0TwNPALclmgdpmAHc/DOw3s8uSs94LbCXA+zppH3CdmRUn/72PbHeg93fSePt2LfCp5Nku1wHdI0MzaXP3rHoA7wd2Am8Af5bpeqZoG99F4k+tTcDG5OP9JMaUnwR2JZ9nZLrWKdr+9wCPJV8vBF4G2oCfAQWZrm8Ktnc50Jrc378AqnJhXwN/DmwHXgd+BBQEbX8DPyFxjGCYRA/8M+PtWxJDLg8ks20ziTOAzml9+um/iEhAZNuQi4iIjEOBLiISEAp0EZGAUKCLiASEAl1EJCAU6CIiAaFAFxEJiP8PiSlsGqZTtScAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.plot(loss_scores)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deXwUVbr/8c+TnYQEAklYsgCBAAk7hEVERUEFF0BEBfcZFWdGrzp6Z0Ydr844d37X2cfdQUVxVBYVkEEUFVERZQm7hC0LJCFAdrInvZzfHx0xxoQ00Emnu5/368VLqup01VMWfKmcPlVHjDEopZTyfH7uLkAppZRraKArpZSX0EBXSikvoYGulFJeQgNdKaW8RIC7DhwVFWX69u3rrsMrpZRH2rZtW5ExJrq5bW4L9L59+5KWluauwyullEcSkSMtbdMuF6WU8hIa6Eop5SU00JVSyktooCullJfQQFdKKS+hga6UUl5CA10ppbyE28ahK6VUh5OzCTLWtf1xBk2D2DEu360GulJKAexcDKvuBbsVkLY9VnhPDXSllHI5Y+DrZ+CTx6HfRXDDmxAS4e6qzooGulLKd9nt8Mn/wDfPwZBr4Jp/QUCwu6s6axroSinfZK2H9++BPctg3N0w7Snwa5txIuW1FvYfq2D/8XL2Hatg9uhYxvbt5vLjaKArpXxPXSUsuxUy18GUx2HSgyDn3m9usxsOF1c1Cm9HgB8tqznVpkunQFL7RGqgK6WU04oy4J3boayZlxPa6h2/ZjwLo289o93a7IYjxVUcKakmp7ianJJqjhRXk1NSRU5JNbUWOwD+fkJiVBij+0Ry4/gEUnpFMLhXOD0jQhAX/OPRHA10pZT3OboN3roOEBh5U/N33wOnQeJFre6qvNbCzpwyth0pZXtOKTtyyqiss57aHhLoR0K3UBK6hXFBUjSDeoaT0iuCATGdCQn0d+FJtU4DXSnlXTI+haW3QlgU3LICuvc/bXO73XC0rOZU98jx8hqq621U1dnIK63mwIkKjAE/gUE9I5g5sjcj4rvSLyqMPt1CiQ4PbrM77jOlga6U8h67l8HKn0NMMtz0HoT3OLWpzmojPb/81J12Tkk1RRX1FFXWYbUbwHEj3z0smM7B/oQGBdAjIoTpQ3sxpk8kI+K7EB4S6K4zc4oGulLKO3z9HHz8W6p6ncdXqc+Qua2C3JITHCl29HEfO1lDQ24T360TA6I7k9wzgqjwYOIjQ0nuFc7AHuGEBXtuLHpu5Uop32WzcnLHSvILCjh2spbAEzu5oOx91tjH80D2z6jPPgRA97AgErqHMrZvJAndYknpHcHohEhiIkLcfAJtQwNdKeUxSqvq+XBnNv2/uI/x9ZvoAiQ3bPssfCb7hz7M33t1JTGqMwndQ+nswXfbZ8O3zlYp5TEsNjtfHixk/YECDhdVc6SkiorSQhYE/o1Uv4Ns6P8Q4SNnkRjdmYiwMC4J78El7i7azTTQlVJta+8KyPrCqaYGKK6sI7vIMc67zmJnuL8wqVMgnYMDSO66h8i6fGT2Qi4YOrtt6/ZATgW6iEwDngb8gVeMMU812Z4ALAK6NrR52BizxsW1KqU8iTHw5V9g/R8hpAv4N/+OFANY7YZ6q506qw1jN/QFBgX4EdzZnyB/P8e7D61Ap3C47l2nxo/7olYDXUT8geeBS4E8YKuIrDLGpDdq9hiwzBjzooikAGuAvm1Qr1LKE9ht8OGvYesrMHwuzHwO/L8f8ldntfHutjxW7zrGztwyaiw2AMb368bs0bFMH9aLiA4+RLAjcuYOfRyQYYzJAhCRJcBMoHGgG+C79012AfJdWaRSyo1KsmD1L6Gq2PnP1FdCaTZMvA+m/v7US69qLY4gf2F9BvknaxncM5wbxsYzpuHdJj27eOfok/biTKDHArmNlvOA8U3a/A74WET+CwgDpja3IxGZD8wHSEhIONNalVLt7dguePNasFmgz/ln9tlJD8CY28kuqmLdvhNsOFTEluwSaiw2xvSJ5E9zhjNpQFSHecrSGzgT6M393zZNlucBrxtj/iYi5wH/FpGhxhj7Dz5kzAJgAUBqamrTfSilXM1a5+jLbsovAPyb+etvszi6SwByvoGltzj6v2//AKIHOX3Ywoo61uw5xvLnN7IrtwyA/tFh3DA2nsuG9OC8xO4a5G3AmUDPA+IbLcfx4y6VO4BpAMaYb0QkBIgCClxRpFLqDFnrYNV/we6lzW8P6QJzFsKARj9Mb1sEa34Ftrrv10Unw83vQZfYFg9Va3G88+RwUTVbD5ew4VAR6cfKAUjuFcGjVwzmimG9iIsMdcWZqdNwJtC3Akki0g84CswFbmzSJgeYArwuIslACFDoykKVUk6qq4ClN0PW5zD2Lojo/eM23y6Ht2+AWS/CsOu+H43S7yJInOxoE9gJRsyFTpE/+rjVZmfVrnz+9UUWBwsqTv0QEOgvjE6I5FeXD+KSwTEk9/LMqdw8VauBboyxisi9wFocQxIXGmP2isiTQJoxZhXwEPCyiPwSR3fM7cY093OeUqpNVRbAW3Pg+LeOsB7Z9N6rwdg7YMlNsPwu2P4GHN7Q7GiUxuqtdjILK9meU8orG7LJLqoiuVcED0wZSJ/uocR3C2VwT89+F4qnE3flbmpqqklLS3PLsZXySiVZ8O/ZUHEcrl8EAy8/fXtLLayYD+nv/2g0CkBNvY3N2cV8daiIrzOLOVRQgcXmyIvkXhE8MDWJS5N74OenfeHtSUS2GWNSm9um/5Qq5Q2O7YI354DdArf9B+LHtv6ZwBCY8zqUZEJUEgDGGHbmlvHW5hxW786n1mInyN+P1L6R3HlBIsm9IkjuGc6AmM76pWYHpIGulKfL+sLRfRLSBW5ffUajUfDzg6gkKuusvL/zKG9tyiH9WDlhQf5cMyqOaUN7Mq5vNzoFte/MO+rsaKAr5cm+XQ4r7oZu/VsdjdJUcWUd23PK+PxAAe/vzKeyzsrgnuH876yhzBoV63NvKvQGesWU8lSbFzger0+YAPMWNzsapSmrzc5bm3NY9PVhsoqqAAgO8OPK4b24eUIfRsV31a4UD6aBrlRHUl8Nn/0BynJaaVfpGJY4+Cq49hXHEMNWbMoq5ner9rL/eAXj+nY79cj90Ngu7T6ZsWobGuhKdRTVJfD29ZCXBjEpzc9U39h59zpGpjT3xGcje/JO8vS6g3y6r4DYrp146ebRXD6kp96JeyENdKU6gpN5jiGHpYfh+jcgZcY57a6yzsqmzGIWb8lh3f4CunQK5KFLB3LnBYn6BacX00BXyt1qyuDVy6GuHG5ZDn0nndVusgorWbv3BOsPFLD9SClWu6FraCD/fdlAbpvYt8PPWK/OnQa6Uu624W9QfhTu/BTimn1epEXZRVWs3HGUNXuOcaigEoAhvSO468JELhgQxeg+kdo/7kM00JVyp9IjsPklGDHP6TCvtdhYseMoy9Jy2ZFThohjYoibxqdw2ZCe9O7a+hekyjtpoCvlTp/9AcQPLnms1aYlVfX8+5sjvPHNYYqr6hnUI5xHpg9m5shYnRhCARroSrnP0e2w5x244KHTPhCUXVTFq19l8U5aHnVWOxcPimb+hf2ZkNhNR6qoH9BAV6q9GAPZX0J1kWN500sQGgXnP9CkmSGrqIoNBwtZf6CQLw8VEujnxzWjYrnzgn4k9Qh3Q/HKE2igK9Vedr4N7//ih+uufgZCIiitqmdjZhEbDhax4VAh+SdrAejTPZR7Jg/g1ol9iAnXbhV1ehroSrWH754AjR0Ds16k3mZn3aEK1mcHs+3zz8ksdDyGHxESwMT+Ufzi4iguSIqiT/cwNxeuPIkGulLtYdPzUHEM5izki9Ju/P4/e8kqrKJraCBjEiKZPTqO8/p3Z3hsFwL8/Vrfn1LN0EBXqq1VFsBX/6Q6cRr3fxHEJ+lb6Ns9lFdvS+WSwTH6xaZyGQ10pdqY9bP/Qyw1zDp4GXl+Rfx62iDumNSP4AB94Ee5lga6Umcr6wv4/CmwW3+0yeCYwq2spp6Yir28aZ1C8tDRvDE9WceMqzbjVKCLyDTgaRyTRL9ijHmqyfZ/ABc3LIYCMcaYrq4sVKkOxVIDK38BxgbRg79fbTccO1nD8ZO11FrsiEBW2GSGzfwTtw3u78aClS9oNdBFxB94HrgUyAO2isgqY0z6d22MMb9s1P6/gFFtUKtSHcemF6E8zzF/Z78LyS+rYcGXWSzdmkuNxcakAVHMGNGbKckxdO8c7O5qlY9w5g59HJBhjMkCEJElwEwgvYX284AnXFOeUh1QVRF89Q8YOI2TPc7jhQ/38frGw9iNYcYIx8M/yb0i3F2l8kHOBHoskNtoOQ8Y31xDEekD9AM+a2H7fGA+QEJCwhkVqlSH8cWfMfVVrOg+n9//ZT3ltRauGRnLg5cNJC4y1N3VKR/mTKA3N6bKtNB2LvCuMcbW3EZjzAJgAUBqampL+1Cq4yrKwL71VT4IvJQH19cyaUAUj16RTEpvvSNX7udMoOcB8Y2W44D8FtrOBe4516KU6igKKmp5Jy2PHTml2E4e47elj9HTHsACvxv41y1juCylh44jVx2GM4G+FUgSkX7AURyhfWPTRiIyCIgEvnFphUq1s3qrna8zi3hnWx5rvz2O1W6YEnWSP9f8jnApZ/2of/DOlTN14gjV4bQa6MYYq4jcC6zFMWxxoTFmr4g8CaQZY1Y1NJ0HLDHGaFeK6liMgX3/cTx632ITw6GCSpaX9uOt7M5U1Frp0imQ2yf25Sd9i4n94F4I9oObP2R6bx3EpTomcVf+pqammrS0NLccW/kQuw0+eAi2veZU8xqCWNznD/SZcA3nD4gi5PB6WHYLhEXDLSugu44lV+4lItuMMc1Ob6VPiirvZamF5Xc67s4n/RIm3veDzTtzS/nzRwdIP1ZOfLdO/Hx8NNP2P8JPcx6FkeGQHuB43W1MMtz0HoT3cNOJKOUcDXTlPSoLHI/i15U7lgv3w/E9MO0pmPDzU80Kymt56sP9LN9xlJ4RIfx2zvlcMyrW8ZbDCath6S3fv7e87wUw920I0VEsquPTQFfeoSQL/j0byvO/n87NPwjmLMQMmc2u3DI+P1DAhkNF7Mwtw1+EX0zuzz0XDyAsuNFfg+BwuHEZfPQbxztarvgrBOiTnsozaKArz3dsF7x5rSOAb/8A4scCUFFrYeWOo7z19Ab2H69ABIbHduFnFyVy3Zh4+ka1MHlEQBBc9Y92PAGlXEMDXXVcBz6EtY+CsZ++XcVxCIvGMu8dDtlj2fhlFl8eKmRLdgl1VjtDYyP4v9nDmD60J11Dg9qndqXcQANddVwbn4H6Kkic/IPV1fU2ckursdjs2OyGys7DecnMZvOzh6m3ZQEwIKYzN45P4JpRsQyP0xd/Kt+gga46ppJsyPkapjwOFzwEwMlqCy98nsFr2w9Tb7UT4CeEBvkTHhLIgJjO/GRwOMk9Ixif2I1eXTq5+QSUan8a6Kpj2r0MEKxD5rAlo4i1e4+zcme+40VYo2J58FJ9EZZSTWmgq47HGGw73yY3YgyznjtAWbWFkEA/Lh4Uw31TkvTVtEq1QANddRjGGI4UV/Pluv9wa9lhnrVM4/whUVw9vDcXDowiNEj/uCp1Ovo3RLnd5qxiVu7MZ8OhQvJKa/h/gUupCwjhnl88SGKsPp2plLM00JXbfJNZzD8/Pcjm7BI6BwdwXv/u/Pz83sz9Mg2/wbM0zJU6Qxroql0ZY/gmq5h/fnqILdklxIQH8+yFNqYHbiPAT6AgD+rLYeQ8d5eqlMfRQFdtzmY3HC6uYm9+OW9+c4Qth0voERHME1encFP4DoLevxtsFhA/xwd6Dne8Q0UpdUY00FWbsNsNnx8sYOFXh0k7UkKtxfG0Z4+IYH4/Ywg3jI0nZOdrsPy/IX4czFsCod3cXLVSnk0DXblEvdVOXmk1uQXFRH3zR+qO7Seo3sr9Af507xpEWHAAYUH+hAYF4HcI2F8POd/AwOkwZyEE6Zhypc6VBro6J1abnTc3HeHvnxxEast4NeivJMsh9gcMok90KJFhgfidmmfcBrZG84effz9c8jj46x9DpVxB/yaps/Z1ZhG/X5XOgRMVzOxn5w+Vf6JzVS4VV75M8ug5OnmyUu3MqUAXkWnA0zjmFH3FGPNUM22uB34HGGCXMeZHE0mrDmzzAtjyL5jxHPQ5r/k22Rtgxc+wVZdgsdkZYTesECEozA//E/VIYCjc8h5d+l3YvrUrpQAnAl1E/IHngUuBPGCriKwyxqQ3apMEPAKcb4wpFZGYtipYuZgxsO5J+OrvENAJ/j0L5rwGg6/4Ybu9KzHv3UW+X08+qL2YToH+jOjThZReEY7hhn7+MHwu9Ehxz3kopZy6Qx8HZBhjsgBEZAkwE0hv1OYu4HljTCmAMabA1YWqFtRVQFnu98uRfZ3/gtFmhdX3w443YcztcPFvYfFcWHoTTPsT9J0EgD3zM+Tjx9huBnK//WF+Mm0Uc8Yl0CnI3+Wno5Q6e84EeizQKDHIA8Y3aTMQQEQ24uiW+Z0x5iOXVKhalr8T3poDVYXfr+sSDzcvh+iBp/9sfTW8+xM4+BFc9BuY/AiIwK2rYNmt8OGvTjX1Az6xjWZ5vydZPmcsMREhbXM+Sqlz4kygN/fNlmlmP0nAZCAO2CAiQ40xZT/Ykch8YD5AQkLCGRerGslcD0tvhk6RMPsV8A8ESw188jgsvAxuehfiUpv/bHUJvH0D5G11zJk57q7vtwV3hhuXUrz7Qz7Yls2m7GJMYCiXXH0DL4ztq190KtWBORPoeUB8o+U4IL+ZNpuMMRYgW0QO4Aj4rY0bGWMWAAsAUlNTm/6joJz17Xuw/G6ISoKb34OI3t9vSxjvmCx50dVw8aMQ0uWHnzUGNr3gmFT5utdhyKwfbD5ZbeGFLzJ4fWMgxgzktol9uOfiATp1m1IewJlA3wokiUg/4CgwF2g6gmUlMA94XUSicHTBZLmyUNVg00uOGekTJsK8xdCpyfRq3RLhjo/hrevg48ea30dwF0e3TL/vH68/UV7Lqp35PLc+wzGJxMhYHrxMJ5FQypO0GujGGKuI3AusxdE/vtAYs1dEngTSjDGrGrZdJiLpgA34lTGmuC0L9zmNR6MMvgqufQUCW5hmrXMM3PUZVBxrfnunSAgKY1duGQs3ZpN2uJSjZTUAXDQwmt9MG0xKb51EQilPI8a4p+cjNTXVpKWlueXYHZbNCuv/CNVFP952Mg8yP3OMRrny745hgmep3mrnmXWHeOHzDLp0CuS8/t0ZnRDJhMTuDI3t0voOlFJuIyLbjDHNfkGmT4p2JBmfOO7Aw6LBr8mlET+4+DG48L8do1HO0oHjFdy/ZAf7j1dw3Zg4/ufqFCJCAs+xcKVUR6CB3pHsWgyhUfDgPseoFRdbvj2PR1fsoXNwIK/cmsrUFJ1AQilvooHeUdSUwoEPIfUOl4d5rcXGk6vTeXtzDhMSu/HMvFHEhOtYcqW8jQZ6R/HtcrDVw4i5Lt1tRa2FO15PY8vhEn4+uT8PXTqQAH8/lx5DKdUxaKB3FLuWQHQy9Brhsl2WVtVz22tbSM8v55l5o5gxonfrH1JKeSy9VesIijMhb4tjHk0XPYlZUFHL3AWb2H+8gn/dMkbDXCkfoHfoHcGuxY5RLMOuP+ddGWNYtSufJ/+TTo3Fxuu3j2XigCgXFKmU6ug00N3NboddSyFxMkT0OqddZRVW8vv/pPPFwUJGxHXhT3OGM7inPiCklK/QQHcnux3WPgInc2DqE2e1i5PVFlbvyWfF9qOkHSklNMifJ65O4dbz+uLvpy/SUsqXaKC7i7UOVvwM9i6HCffAkNlnvIsPdh/jsZV7KK22kBTTmV9PG8S1o+Pooa+3VconaaC7Q22549W32V/ApU/CxPvO6MvQ0qp6/uf9b1m9+xgj4rrw+k+GMjyui77aVikfp4He3ioL4M1r4cRemPUijHR+6tWKWgtvfHOElzdkUVVn5VeXD+LuCxN1XLlSCtBAb18lWY53lVeegHlLYOBlTn2sotbC6xsP88pX2ZyssXDJ4Bh+dfkgknvpF55Kqe9poLeX0sPw6uVgtzimeYsf2+pHyhuC/NVGQX7/lCRGxHdt9bNKKd+jgd5eNi9wvK/l5xshetBpmxpjeHdbHn9cs4+yagtTk2O4b0oSw+M0yJVSLdNAbw82K+xZBoOmtRrm2UVVPLp8D99kFZPaJ5Inrh7CsDh9R7lSqnUa6O0hcx1UFcKIeT/aVFNvY+HGbHbllrH/eAU5JdWEBwfwx2uGMm9sAn46llwp5SQN9PawazF06gYDLv3B6oKKWu5alMauvJMkRocxLLYL16fGcV1qvI4lV0qdMQ30tlZTBvvXOKaOCwg6tfrA8Qp++vpWSqrqefnWVC7VySaUUufIqQHMIjJNRA6ISIaIPNzM9ttFpFBEdjb8utP1pXqo9JVgqzv1nvOjZTX8de0B5rz4NRabnWV3n6dhrpRyiVbv0EXEH3geuBTIA7aKyCpjTHqTpkuNMfe2QY2ebediTPRg1pf35q1PtrL+QAEGmDI4hidnDqV3107urlAp5SWc6XIZB2QYY7IARGQJMBNoGuiqiYpjBwnP3cSLAbfw50XbiOoczC8mD2DuuHjiIkPdXZ5Syss4E+ixQG6j5TxgfDPtrhWRC4GDwC+NMblNG4jIfGA+QEJCwplX60G255SSvehxZho/0qMu5/kZo7lsSA8C9TF9pVQbcSbQmxs3Z5os/wdYbIypE5GfAYuAS370IWMWAAsAUlNTm+7DK9jshpe+yGTFJ5/zUdDHlCTfyHNzr3Z3WUopH+BMoOcB8Y2W44D8xg2MMcWNFl8G/nTupXmeOquN+xfv5KO9x3m/+wr8LZ2Iuep37i5LKeUjnPn5fyuQJCL9RCQImAusatxARBpPtTMD2Oe6Ej1DTb2Nu97Yxkd7j/PsxBpGVH2FTHoAOke7uzSllI9o9Q7dGGMVkXuBtYA/sNAYs1dEngTSjDGrgPtEZAZgBUqA29uw5g6notbCHYvS2Hq4hKeuGcrVu38C4b0dE1copVQ7cerBImPMGmBNk3WPN/r9I8Ajri3NM1TVWbn9ta3szC3jnzeMZGbgFji6DWa+AEE6kkUp1X50yMU5qLXYuHNRGjtySnl23ihmjoyFtIUQ2ffUg0RKKdVeNNDPUp3Vxt3/3sam7GL+dv0IrhjWC8pyIXsDjLgR/PzdXaJSysdooJ8Fi83OfYt38MXBQv44axjXjIpzbNi9FDAw/Hq31qeU8k0a6GfIZjc8tGwXa/ee4PGrUrhxfMMDUsbAriWQMBG69XNvkUopn6SBfgbsdsOjy/ewalc+v542iJ9OahTcR7dD8SEY+eN3niulVHvQQD8Df//kIEvTcrlvShK/mDzghxt3LYaAEEiZ6Z7ilFI+TwPdSUWVdby8IYuZI3vzy6lJP9xorYNv34XBV0KIThenlHIPneDCSW98fZh6m537piQhIlCWA/tWOzaWHnZMAN3MFHNKKdVeNNCdUF1v5Y1NR7g0uQf9ozs7Jn1+81ooOvh9o8h+kHix+4pUSvk8DXQnLNuaS1m1hbsvSnSs2L7IEeZzFkL/KY51QWHgr/87lVLuownUCqvNzssbskntE8mYPt2grgI+/z/H8MQhs0Gae7uwUkq1P/1StBUf7DnG0bIa7r6ov2PFxqehqhAu+18Nc6VUh6KBfhrGGF7ekEX/6DCmDI6B8nz4+jnHnXncGHeXp5RSP6BdLqex9XApM068yI3hO/B7OgDqK8HYYOoT7i5NKaV+RAP9NL76dCUPBnyALeZC6NrwvpZB0xxvU1RKqQ5GA70FeSWVTM19lvKQGCJuXgaBndxdklJKnZb2obdg2wevMtwvC9vk32qYK6U8ggZ6M6qrK0nNfJbc4AFETrjF3eUopZRTNNCbsf/9vxFLIdUXPaETVSilPIZTgS4i00TkgIhkiMjDp2k3R0SMiKS6rsT2Zax1JB58le2Boxh43tXuLkcppZzWaqCLiD/wPDAdSAHmiUhKM+3CgfuAza4usj1lfL2SruYklSPucLyESymlPIQzd+jjgAxjTJYxph5YAjT30u8/AH8Gal1YX7ur3voWxSaCMZfMcXcpSil1RpwJ9Fggt9FyXsO6U0RkFBBvjFl9uh2JyHwRSRORtMLCwjMutq2VlxSQXL6R/dGXExaqI1uUUp7FmUBvrt/BnNoo4gf8A3iotR0ZYxYYY1KNManR0dHOV9lO9n3yOkFiJXrS7e4uRSmlzpgzgZ4HxDdajgPyGy2HA0OBz0XkMDABWOVpX4waY4g4+C6H/fqQNHyiu8tRSqkz5kygbwWSRKSfiAQBc4FV3200xpw0xkQZY/oaY/oCm4AZxpi0Nqm4jRxI30Gy7QDFSbMRPx3NqZTyPK0mlzHGCtwLrAX2AcuMMXtF5EkRmdHWBbaX/C9ew2aEgVN/6u5SlFLqrDj1LhdjzBpgTZN1j7fQdvK5l9W+austDD6xhkOdxzI4OsHd5Sil1FnRvgVg11cf0FuK8Bs5192lKKXUWdNAB2w73qaSTvS/UANdKeW5fD7QK8rLGFH+BQe7T8E/OMzd5Sil1Fnz+UDf//kSwqSWsHE3ubsUpZQ6Jz4f6J3Sl5EvMSSlXu7uUpRS6pz4dKCXHT9Ccs12snpfhZ+/viZXKeXZfDrQs9e/hr8YYs6/zd2lKKXUOfPpQI/KXM63foNJSh7h7lKUUuqc+WygFx3NJN56hMI+0/W950opr+Czgb53yzoAEkdPdXMlSinlGj4b6JUZ31BHEAnJY91dilJKuYRPBnpRZR09K/ZQGJ6MBAS7uxyllHIJnwz0T3bnMFSy6dRvgrtLUUopl3HqbYveZv+OrwgWK0GDz3d3KUop5TI+d4deXFlHwLFtAEjcODdXo5RSruNzgf5x+glGySHqO8dCRC93l6OUUi7jc4G+Zs8xUgMyCeyjd+dKKe/iU4FeVl1PZuYheppC7W5RSnkdnwr0z/YXMJxDjoV4DXSllHdxKtBFZJqIHBCRDBF5uJntPxORPSKyU0S+EpEU15d67j7ee4JJIVkY/yDoOczd5SillEu1Gugi4g88D7LKCc8AAAnvSURBVEwHUoB5zQT228aYYcaYkcCfgb+7vNJzVGux8eWhQi7olI30Ggn6QJFSyss4Mw59HJBhjMkCEJElwEwg/bsGxpjyRu3DAOPKIs9JfTWsfoDKwuM8Z04SX7Mfhs13d1VKKeVyzgR6LJDbaDkPGN+0kYjcAzwIBAGXNLcjEZkPzAdISEg401rPTuY62L0US3AiMX4CvUfCsDntc2yllGpHzvShN/du2R/dgRtjnjfG9Ad+AzzW3I6MMQuMManGmNTo6Ogzq/RsZazDBIUzy/r/eGnQK/jd+SnEjm6fYyulVDtyJtDzgPhGy3FA/mnaLwFmnUtRLmMMZK6jrMcETlTZuWxIT3dXpJRSbcaZQN8KJIlIPxEJAuYCqxo3EJGkRotXwndjA92sJAvKctjsN5JAf2HyoHb6qUAppdyg1T50Y4xVRO4F1gL+wEJjzF4ReRJIM8asAu4VkamABSgFOsYknRmOSSzeKEhkQmJ3IkIC3VyQUkq1HafetmiMWQOsabLu8Ua/v9/FdblG5jrqI/rwdUEX/nBhD3dXo5RSbcp7nxS11kP2BtJDx+InMG2ovohLKeXdvDfQczeDpYplpUlM7B9FdLg+SKSU8m7eG+iZ6zASwKqT/bl6hN6dK6W8n/cGesY6csKGUucfxrQhGuhKKe/nnYFeWQjHd/NhTQoXDYymS6iOblFKeT/vDPR9jmHyq6qHcvWI3m4uRiml2od3ThK9awnHQxLJsvVjarIOV1RK+Qbvu0MvyoC8LSypO5+pyT0JC/bOf7OUUqop7wv03Usw4sfbNRO0u0Up5VO86/bVboddS9kfOoY6YvTdLUopn+Jdd+hHNsLJHF6tGM+Vw3sRHODv7oqUUqrdeFeg71qCJSCM1fVjmD0q1t3VKKVUu/KeQK+vhvSVbAyaREy3SMb0iXR3RUop1a68J9APrYX6Sl4qG8usUbGINDfRklJKeS/vCfSMddQFdGarfRDXaHeLUsoHeUegGwOZn7FFhjMsvjv9osLcXZFSSrU77wj0wgNQfpQPqlOYPVrvzpVSvsk7Aj3TMdXcBtswputEFkopH+Ulgf4ZR/3j6B43QCeyUEr5LKcCXUSmicgBEckQkYeb2f6giKSLyG4RWScifVxfagsstZjDG/m4bggXDdQnQ5VSvqvVQBcRf+B5YDqQAswTkZQmzXYAqcaY4cC7wJ9dXWiLcr5GrDV8aR+uga6U8mnO3KGPAzKMMVnGmHpgCTCzcQNjzHpjTHXD4iYgzrVlnkbGOqwSSHrQMEbGd223wyqlVEfjTKDHArmNlvMa1rXkDuDD5jaIyHwRSRORtMLCQuerPA2T+Rk7GExqUjwB/t7xlYBSSp0NZxKwuUcuTbMNRW4GUoG/NLfdGLPAGJNqjEmNjnZB90j5MaQgnU/rh3KRvllRKeXjnAn0PCC+0XIckN+0kYhMBX4LzDDG1LmmvFZkfALABvsw7T9XSvk8ZwJ9K5AkIv1EJAiYC6xq3EBERgH/whHmBa4vswW7lpLvH4vpMYweESHtdlillOqIWg10Y4wVuBdYC+wDlhlj9orIkyIyo6HZX4DOwDsislNEVrWwO9cpPQJHvmJJ3UQuGhTT5odTSqmOzqkZi4wxa4A1TdY93uj3U11cV+t2LwPgPesk/qb950op5aFT0BkDuxZzIGQEdQFx+u5zpZTCUx/9z0uDkkwWVk5g9uhYAnW4olJKeegd+q63sfqF8IF1LCvGtN8zTEop1ZF53q2ttQ7z7Xt87j+epITeJPUId3dFSinVIXheoB/8CKk9yaKq87huTHzr7ZVSykd4XqDbLBwOHcY2/+FcNULffa6UUt/xuD70mkHXcPW74Uwb2oOIkEB3l6OUUh2Gx92hr917nIo6K3NS9ctQpZRqzOMCvXNwAJem9GBCv+7uLkUppToUj+tymZrSg6kpPdxdhlJKdTged4eulFKqeRroSinlJTTQlVLKS2igK6WUl9BAV0opL6GBrpRSXkIDXSmlvIQGulJKeQkxxrjnwCKFwJGz/HgUUOTCcjyFL563L54z+OZ5++I5w5mfdx9jTLPzbrot0M+FiKQZY1LdXUd788Xz9sVzBt88b188Z3DteWuXi1JKeQkNdKWU8hKeGugL3F2Am/jiefviOYNvnrcvnjO48Lw9sg9dKaXUj3nqHbpSSqkmNNCVUspLeFygi8g0ETkgIhki8rC762kLIhIvIutFZJ+I7BWR+xvWdxORT0TkUMN/I91dq6uJiL+I7BCR1Q3L/URkc8M5LxWRIHfX6Goi0lVE3hWR/Q3X/Dwfuda/bPjz/a2ILBaREG+73iKyUEQKROTbRuuavbbi8ExDtu0WkdFnejyPCnQR8QeeB6YDKcA8EUlxb1Vtwgo8ZIxJBiYA9zSc58PAOmNMErCuYdnb3A/sa7T8J+AfDedcCtzhlqra1tPAR8aYwcAIHOfv1ddaRGKB+4BUY8xQwB+Yi/dd79eBaU3WtXRtpwNJDb/mAy+e6cE8KtCBcUCGMSbLGFMPLAFmurkmlzPGHDPGbG/4fQWOv+CxOM51UUOzRcAs91TYNkQkDrgSeKVhWYBLgHcbmnjjOUcAFwKvAhhj6o0xZXj5tW4QAHQSkQAgFDiGl11vY8yXQEmT1S1d25nAG8ZhE9BVRHqdyfE8LdBjgdxGy3kN67yWiPQFRgGbgR7GmGPgCH0gxn2VtYl/Ar8G7A3L3YEyY4y1Ydkbr3ciUAi81tDV9IqIhOHl19oYcxT4K5CDI8hPAtvw/usNLV/bc843Twt0aWad1467FJHOwHvAA8aYcnfX05ZE5CqgwBizrfHqZpp62/UOAEYDLxpjRgFVeFn3SnMa+o1nAv2A3kAYji6Hprztep/OOf9597RAzwPiGy3HAfluqqVNiUggjjB/yxizvGH1ie9+BGv4b4G76msD5wMzROQwjq60S3DcsXdt+JEcvPN65wF5xpjNDcvv4gh4b77WAFOBbGNMoTHGAiwHJuL91xtavrbnnG+eFuhbgaSGb8KDcHyJssrNNblcQ9/xq8A+Y8zfG21aBdzW8PvbgPfbu7a2Yox5xBgTZ4zpi+O6fmaMuQlYD8xpaOZV5wxgjDkO5IrIoIZVU4B0vPhaN8gBJohIaMOf9+/O26uvd4OWru0q4NaG0S4TgJPfdc04zRjjUb+AK4CDQCbwW3fX00bnOAnHj1q7gZ0Nv67A0ae8DjjU8N9u7q61jc5/MrC64feJwBYgA3gHCHZ3fW1wviOBtIbrvRKI9IVrDfwe2A98C/wbCPa26w0sxvEdgQXHHfgdLV1bHF0uzzdk2x4cI4DO6Hj66L9SSnkJT+tyUUop1QINdKWU8hIa6Eop5SU00JVSyktooCullJfQQFdKKS+hga6UUl7i/wOnLTqWRuq4mgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.plot(train_scores2)\n",
    "plt.plot(test_scores2)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MSE VS Cross Entropy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deXhU5d3/8fc3ISEsAQTCGjCgYVdUIkIFt7rgilvFtdVHxbZu1WqLj9aqfbo8drH+LD8UxWrdUNFaRETFouLCEkRQlmjYIyBhCWHNej9/nLGGMEmGMDMnc+bzuq5cmXPmnjPfwwmfnNxzn/uYcw4REUl8KX4XICIi0aFAFxEJCAW6iEhAKNBFRAJCgS4iEhDN/Hrjjh07upycHL/eXkQkIS1YsGCzcy4r3HO+BXpOTg75+fl+vb2ISEIyszV1PacuFxGRgFCgi4gEhAJdRCQgFOgiIgGhQBcRCQgFuohIQCjQRUQCwrdx6CIiTc7qD2Hl+7F/n76joPuQqG9WgS4iUl0NH/wR3vs94ACL7ftldlGgi4hE3Z4S+OcN8OUMOHIMnPNXSG/pd1WNokAXkeS18Qt48UrYvg7O+hMcex1YjM/OY0iBLiLJafHLMPVmyGgLV0+Hnsf5XdFBU6CLSDDtLYVZv4XtRfs/V7YDVr0PPb8HP3gKMjvHvbxYUKCLSPBsWg4vXgFbV0FWv/DdKCNug5PvhtS0+NcXIwp0EQmWJf+E1270Ptj80VTIGeF3RXGjQBeRYKiqhJm/hk/+BtlD4ZKnoU03v6uKKwW6iCS+nZvg5WtgzYdw7PVwxu+gWbrfVcWdAl1EEs+KWfD2r6B8h7e8eytUVcAFj8HgS/2tzUcKdBFJHM7Bhw/Bv38D7Q+DHqGhhqlpcNyPocsR/tbnMwW6iMTWuvmw9pPobGvNR94VnQMvhPMegeato7PdgIgo0M1sFPAwkAo84Zz7Q63newJPA+1CbcY556ZHuVYRSSQ1z6ZddXS2mZoOZ/wehv0koa/ojJUGA93MUoHxwGlAETDfzKY655bWaHYP8JJzboKZDQCmAzkxqFdE/FBVAVXlkbev2AOv3wrLp3ln02f9CZo1P/g6UtOis52AiuQMfShQ6JxbCWBmk4HRQM1Ad0Cb0OO2wPpoFikiPvp8Crz+s+8+gIyUpepsOs4iCfTuwLoay0VA7UkP7gPeNrObgVbAqeE2ZGZjgbEAPXv2PNBaRSSeqirg7Xtg7qPQczj0PfPAXp8zIiZTxErdIgn0cL9aXa3ly4CnnHN/NrPhwDNmNsi5fTvOnHMTgYkAeXl5tbchItH01Uz4YorXl11byw5wwh3Qsv1367atho8ehvLd3nLxMtiwCIbdCKfdH6hL5IMqkkAvAnrUWM5m/y6Va4FRAM65T8wsA+gIbIpGkSJyAKqr4L0/wAcPesGdHmYkSOl6r397zDPQdbAX/q9c6/WTt8ry2jTLgIsmwREXx7d+abRIAn0+kGtmvYCvgUuBy2u1WQt8H3jKzPoDGUBxNAsVkQjs3gqvXg+FM+GoK+HsP0Fai/3bFeXDi1fBpNNh4AWwaDJ0HgRj/gHte8e/bomKBgPdOVdpZjcBb+ENSXzSObfEzB4A8p1zU4GfA4+b2W143TFXOxfu7zwRiZkNi7yQLl0P5zwEQ66p+8PI7Dy44QOYcg0seiHh79QjHvMrd/Py8lx+fr4v7y0SOJ89D9NugxbtvW6U7LzIXldVCZuWeldYaiRKQjCzBc65sAdYV4qKJLLKcpgxDvInQc5IuPjv0Dor8tenNoOuR8auPokrBbpIoipdDy/9EIrmw/duhu/f5wW0JC0dfZGmxDkofNe7aXF9KvfC7D97V2T+4GkYeH586pMmTYEu0lRU7IE3fg6fPRdZ+4594Oo3IKtvbOuShKFAF2kKtq32RqhsXAwn/hKGXE34a/pqaJWlLhbZh34aRPxWugEeP8UbcXLZi9B3lN8VSYJSoIv4bfodUL4Lxr4Pnfr5XY0ksBS/CxBJakunepfgn3SXwlwOmgJdxC97SmD6nd5FPcNv8rsaCQB1uYjEU+l62Lvde/zxI7BrE1w+WR9uSlTop0gkXgrehMmX73s7tuE3Qbej/atJAkWBLhIPe0th2u2Q1R9OvNNbl9YKDjvF37okUBToIvHw7v2wYwOMeRaydRcfiQ19KCoSa2vnwPxJ3r01FeYSQzpDF2mssp3w1dv79omH8/6D0LYHnHx3fOqSpKVAF2kM57yZDle823DblGbeFaDNw9wKTiSKFOgijbH4JS/Mv38v9D+v/rbN20Bm5/jUJUlNgS5yoHZtgbfugu55cPzPICXV74pEAH0oKnLg3vpv7+Kg8x5RmEuTokAXORAFM2DxZBhxG3Qe4Hc1IvtQoItEwjmY8yi8eAV0GgAj7/C7IpH9qA9dksPeUigrbbhd6y77z6tSvgtevxU+fxn6ngUXPAppGbGpU+QgKNAl+D57AabdBpV7Gm6b1c+7mrNjrre8ZYV3J6FNS+GUX8GI2yFFf9hK06RAl+CqLPdGo8x/AnJGwpGX1N++fDd88CBMPBkumOCNH3/1Bi/Ar5wCh58an7pFGkmBLsGxZQXM+h2U7fCWS9ZA8XI4/lY45d7Ipqjtd7Z3wdCLV3rLXQfDJc/AIYfGrm6RKFGgSzAUzIBXxwIO2vf21jXPhEv+AQNGR76ddj3gv2bAzPuhuhJOux/SWsSkZJFoU6BL01W+C9Z83PBcKWs/gQ8fit7ZdLPmMOp3B7cNER8o0KXpevc3MHdCZG2PuhLO/pPOpiWpKdClaSrbAQufhX7nwMjb62+b1gqy+oJZfGoTaaIU6NI0LZoM5Tu8KzK7aw5xkUhoQK00Pc7BvInQ7RjIzvO7GpGEoUCXpmflLNj8JRx3g9+ViCQUBbo0PfMeh1ZZMPACvysRSSjqQxf/OedNRwtQuh4K3oQT7vCGD4pIxBTo4q8dG+Hla2Dtx9+ts1TI+y//ahJJUAp08c+aj+Hlq70hiifd5d2qDaBjH2jTzdfSRBKRAl2iq2IvzP4zbF1Rf7uqCiiYDu0Ohate080iRKJAgS7RU7LWm9hq/UJof1jDF/oMvNC7ujOjbXzqEwk4Bbp4qiph22roeHj97TZ+8d0HmDXt2ADT7/QmtLr0eW/WQhGJq4gC3cxGAQ8DqcATzrk/hGlzCXAf4IBFzrnLo1inxNLOYphyDayeDcf9GE7/H0hN27dNZTnMGAf5k+reTlY/GPNcw78URCQmGgx0M0sFxgOnAUXAfDOb6pxbWqNNLnAXcLxzbpuZdYpVwRJl6+Z73SR7tkL/82Duo7BhEfzgKcjs4rUpXe+1KZoPw2+C3NP3346leJfop7eMa/ki8p1IztCHAoXOuZUAZjYZGA0srdHmemC8c24bgHNuU7QLlTD2lMCbv4ANi79b13MYnPFbSG9V/2udg/wn4c1feiNKrn0Huh4Jn0+BqTfD+KGQGRppUroeXNWBzy0uInEVSaB3B9bVWC4CjqvVpg+AmX2E1y1zn3NuRu0NmdlYYCxAz549G1OvfOubJd5ddUrWQu4ZkJIKVeWw4ClYNw/GPAMdDgv/2oo9MO12WPS8d1u1Cx+Hlu295464GDr1hw//CpV7vXVdB3szHmb1jcuuiUjjRBLo4YYquDDbyQVOArKB2WY2yDlXss+LnJsITATIy8urvQ2J1Ldn0c3bwNVveGfl3yqcCa9c590Xc+RtkN5639c6BwufgY2L4cRfwonj9r/pceeBcNHjsd8PEYmqSAK9COhRYzkbWB+mzRznXAWwyswK8AJ+flSqFE9VBbx9j9fP3XP4vv3c3zr8VBj7vtfnPfO+8NvJaAeXvQh9R8W6YhGJo0gCfT6Qa2a9gK+BS4HaI1heAy4DnjKzjnhdMCujWWjScM77qm3Xpu8ukR/2Uzjtgf1HonzrkEPh+lneB53hpLeGtIzo1SwiTUKDge6cqzSzm4C38PrHn3TOLTGzB4B859zU0HOnm9lSoAq40zm3JZaFB1LpBpgwHPZsC/98Wku4aJLXz92QlBRo1TG69YlIkxbROHTn3HRgeq1199Z47IDbQ1/SWPlPeiNXTrgTUmqdfVsK9D8XOvXzpzYRafJ0pWhTUVkGC/4Ofc6AU+7xuxoRSUC6wUVTseQ12FUMQ8f6XYmIJCgFelMx7zHokAu9T/a7EhFJUAr0pqBoAXy9wDs7rz0mXEQkQkqPpmDeY5CeCUdd5nclIpLAFOh+27AIlvwTjrocmmf6XY2IJDAFup8+ex4mne7d4f57N/ldjYgkOA1b9ENlGcy4y5tbPGckXPx3aJ3ld1UikuAU6PG2/WtvnpWv8+H4W+GUeyFVh0FEDp6SJJ5WfeDNx1K5V3OLi0jUKdDjZfWH8I/zvTnKxzyrucVFJOoU6PHy4V+9ybKuexcy2vhdjYgEkEa5xMOWFVD4Dgy5RmEuIjGjQI+H+U9ASjPIu8bvSkQkwBTosVa2ExY+CwPO3//uQiIiUaRAj7XFk6GsFI67we9KRCTgFOix5BzMexy6HgXZx/pdjYgEnAI9llbOguLl3tm5md/ViEjAKdBjpbIM3hwHbXvAwAv9rkZEkoDGocfK7L/A5gK4YgqkZfhdjYgkAZ2hx8Km5TD7zzDoYsg9ze9qRCRJKNCjrboaXr8FmreGUX/wuxoRSSLqcom2z56DdXPh/AmaEldE4kpn6NHkHHzyN+g6GAbrdnIiEl8K9Gha9X5omOKPNUxRROJOgR5N8x6Hlh00TFFEfKFAj5aStVAwHY75kYYpiogvFOjRMv8JwODYa/2uRESSlAI9Gir2wKf/gH5nQ9tsv6sRkSSlYYuNVTgTFjztPd69BfZs04yKIuIrBXpjlK6Hl672+spbdvTWDboYDj3e17JEJLkp0A+Uc/DGHVBdCde+A+17+V2RiAigPvQDt2wqFLwBJ9+lMBeRJkWBfiD2lMD0O6HLkTDsRr+rERHZh7pcGvL2PVDwpve4bCfs2gyXvwSp+qcTkaZFqVSfZdPg40cgZyS07uSt6zMKuh3lb10iImEo0OuydztMvwM6D4Kr/gmpaX5XJCJSLwV6XWbeDzu/gUufU5iLSELQh6LhrPkE8id5syZ2H+J3NSIiEYko0M1slJkVmFmhmY2rp93FZubMLC96JcaZc15XS9uecPLdflcjIhKxBgPdzFKB8cCZwADgMjMbEKZdJnALMDfaRcbVqg/gmy/gpHHebeRERBJEJGfoQ4FC59xK51w5MBkYHabdb4AHgb1RrC/+5k305jQfdJHflYiIHJBIAr07sK7GclFo3X+Y2dFAD+fctPo2ZGZjzSzfzPKLi4sPuNiY05zmIpLAIgn0cPdSc/950iwFeAj4eUMbcs5NdM7lOefysrKa4A2UNae5iCSwSAK9COhRYzkbWF9jORMYBLxnZquBYcDUhPtgVHOai0iCiyTQ5wO5ZtbLzNKBS4Gp3z7pnNvunOvonMtxzuUAc4DznHP5Mak4Vj5/WXOai0hCazDQnXOVwE3AW8Ay4CXn3BIze8DMzot1gXHhnPdhaKeBmtNcRBJWRFeKOuemA9Nrrbu3jrYnHXxZcbZ2Dmz8HM59GCzcRwYiIk2frhQFmPcYZLSFIy7xuxIRkUZToJeuh6VT4eirIL2l39WIiDSaAj3/SXDVcOx1flciInJQkjvQK8tgwVPeHOe6nZyIJLjkDvQlr8GuYjhurN+ViIgctOQO9HmPQYdc6HWS35WIiBy05A30krXw9QIY8iNISd5/BhEJjuRNslWzve+9T/a3DhGRKEneQF8925smt9N+U7uLiCSk5Ax057wz9JwR6m4RkcBIzjTbuhJKiyBnpN+ViIhETXIG+upQ/3mvE/2tQ0QkipIz0FfNhtadoWOu35WIiERN8gW6c94Zes5IzawoIoGSfIG++UvY+Q30OsHvSkREoir5An3VB973XvpAVESCJfkCffVsaJMNh2gyLhEJlojuWJTQqqtgzgTYW+Itr3wP+p6t/nMRCZzgB/qKf8PbdwPmhXhKGgwY7XdVIiJRF/xAX/Y6pLeGO1dAWobf1YiIxEyw+9Crq2D5G5B7usJcRAIv2IG+bi7s3gz9z/G7EhGRmAt2oC97HVLTvTN0EZGAC26gOwfLpsFhp0DzTL+rERGJueAG+oZFsH0t9D/X70pEROIiuIG+7HWwFOhzpt+ViIjERXADffk0OPR4aNXB70pEROIimIFeXADFy6H/eX5XIiISN8EM9PwndUWoiCSd4AV62Q5Y+BwMPB8yO/tdjYhI3AQv0BdNhvIdMPQGvysREYmrYAW6czDvceh2NGTn+V2NiEhcBSvQV74Hmwu8s3NNjysiSSZYgT5vIrTsCIMu9LsSEZG4C06gb1sDBW/CkKuhWXO/qxERibvgBPoXUwDnBbqISBIKTqAvmwbdh0C7Hn5XIiLii2AEesk6WP+pJuISkaQWjEBf/ob3vZ8CXUSSV0SBbmajzKzAzArNbFyY5283s6VmttjM3jWzQ6Nfaj2WT4Os/tDx8Li+rYhIU9JgoJtZKjAeOBMYAFxmZgNqNVsI5DnnjgSmAA9Gu9A67doMaz7SbeZEJOlFcoY+FCh0zq10zpUDk4F9Zr1yzs1yzu0OLc4BsqNbZj0K3gRXrf5zEUl6kQR6d2BdjeWi0Lq6XAu8Ge4JMxtrZvlmll9cXBx5lfVZ9jq06wldjozO9kREElQkgR7uGnoXtqHZlUAe8MdwzzvnJjrn8pxzeVlZWZFXWZeyHbBylvdhqC71F5Ek1yyCNkVAzcHd2cD62o3M7FTgbuBE51xZdMprQMEMqCpX/7mICJGdoc8Hcs2sl5mlA5cCU2s2MLOjgceA85xzm6JfZl2VPQGH9IIew+L2liIiTVWDge6cqwRuAt4ClgEvOeeWmNkDZvbtPd7+CLQGXjazz8xsah2bi54Ni2DdHBh6PaQEYzi9iMjBiKTLBefcdGB6rXX31nh8apTratjciZDWEo66Iu5vLSLSFCXmqe2uLfD5y3DkGGjRzu9qRESahMQM9E+fhqoyGDrW70pERJqMxAv0qkqYPwlyRkLn2hesiogkr8QL9C/fhNIiOE43gRYRqSnxAr2qHHoOhz5n+l2JiEiTEtEolyZl0EXel4iI7CPxztBFRCQsBbqISEAo0EVEAkKBLiISEIn3oaiISIzM/qqYjwq3NPr1mRnN+OHwQ8nMSPvPup1llUyet5azj+xK17YtolFmnRToIpL0qqodD73zJX+bVUizFCMlpXH3VyivrOaVT4t47Moh5HbOpHDTTm54Jp8Vxbt49P0VPHLZMQw/rEOUq/+OORf2XhUxl5eX5/Lz8315bxEJb8vOMvZWVtfbJtWMzm2aYwl8U5m9FVVs2VUOQFlFFfe9vpQPvixmTF4P7h89kIy01EZtd+7KLdz4/EJ2l1dyzfE5PPXRajLSUhl3Zj8mvL+CNVt2M25UP64b2avR/35mtsA5lxfuOZ2hiwiVVdX88a0CHvtgZUTtT+6bxV/HHE3blmkNN25i5q/eyk+f+5TiHd/dhyc9NYXfX3gElw3teVDbPq53B6bdPIKfPreA8bNWMLhHOx698hi6tm3BqEFduPPlxfx2+jIcjrEnHHawu7IfnaGLJLnNO8u4+fmFfLJyC2PyejDk0EPqbb9++x7Gzyqka9sWPHrlEAZ0axOnSg+Oc46/f7Sa301fRo/2LRl7Qm9SQ2fJR/VsR5/OmVF7r/LKamZ/VcyI3I40b/bd2b5zjmfnrOG8wd0b/cuwvjN0BbpIAquudjz50So+Ktzc6G0sWV/K9j0V/O6CI7hoSHZEr1mwZhs/fW4B2/dUMKx3h7A3Ho5ETsdW3HlGX1qmH1hnwYrinTz0zpfsKquM+DUleypYuLaE0wZ05s+XDKZNRuL9dQEKdJFAKt1bwe0vLmLmsm/I7dSaFumN6/fNzGjGXWf2Z1D3tgf0uuIdZdz3+hLWbd3dqPd1Dpas305up0wevWoIvTq2iuh1M77YwB0vLybFvF8IkTLgrCO6cv3I3o3+0LMpUKCL+KSyqpq5q7ayt6Jqv+fSUlMY1rsD6c0O/HKQgo07uOGZfIq27eHus/tz9fdyEvJDytlfFXPLCwuprHLcdVZ/OrdpXm/7OSu38PjsVQzu0Y4JVxxDt3axHQbYFCnQRXyweWcZt7ywkI9X1D2u+eie7fj/VxxzQOOTpy5azy+nLKZ1RjPGX34MQ3u1j0a5vinatpufPPspn3+9PaL2lx/Xk1+fO2CfvulkokAXibPP1pXwk2cXsHVXOb86ZwBHZu/fnfHlNzv59b++oEV6akTjkyuqqvn99OU8+dEqjs05hPGXH0OnNhmx2oW4qqiqpmDjDqobyKOW6c04vFPrOFXVNGnYojRZeyuq+O0by5i57Bu/S4mq4h1ldGmbwSs/+V6dfdNHZrfjqB5tGfvMAq54Yg6dGwjnvRVVbNtdwTXH5/DfZ/UnLTU4M3ekpaYccB++7E+BLr6p+af2mYO6kJkRnB/Hdi3T+cmJh3FIq/R62x3eKZN/3Xg8E95bweadZfW2BTilXydGDeoarTIlYBLuf9DiohLeXbaJ207r43cpgbJpx15eW/g1FVXx6YKrqKrm6Y9XU1nlmHjVEE4f2CUu79sUZWak8YtR/fwuQwIg4QJ94doSHn73K84d3C3p+9KiJdyVc/HQv2sbxl9+NL2zdBxFoiHhAv30gZ359dQlvLVkI4d3Ojyi1+ytqKKswpufIjXVaN28/t2urKpmV9n+w8yC6NWFRfz2jWVkH9KCp28ZGddfkmmplpBD7USaqoQL9K5tWzC4RzveXrKRG09uONA/L9rO5Y/PYUeNK8ouycvmgdGDwk7A89m6Em587lO+LtkT1bqbslP7d+YvYxL3yjkR8SRcoAOcMbAzD84oYH3JnnovLKioquaXrywmIz2Vn53WBwPWbNnF05+sYemGUiZcMYQe7VsC3hwLz89by/1Tl9KpTXPuObs/KUlw9piV2Zyzj+ia0FfOiYgnIQN91MAuPDijgLeXbOTq43vV2W7Sh6tCwX0MZx7x3ciAkblZ3PbSZ5z7tw8ZcXhHALbtLuejwi2c2CeLhy89inYt6x+dICLS1CTkQNbeWa3J7dSaGUs21tlmzZZdPPTOl5w+oDOjBu07guLUAZ15/aYRDOzWhqUbSlm6oZSN2/dy26l9ePLqYxXmIpKQEvIMHWDUoC6Mn1XI1l3ltG+VTnW1Y1FRCWWhyfkfnvkV6akpPDB6UNgP3nI6tuK564bFu2wRkZhJ2EA/Y2AXHvl3ITOXfsMZA7vwsxcXMqugeJ82/3P+ILq0Dcal0SIiDUnYQB/YrQ3d27Xg2blr+NusQjZs38M9Z/f/z2T7bTLSdCmxiCSVhA10M2PUoC5M+nAVnds0Z/LY4Q3eaUVEJMgSNtABrh3RCwNuOPEwsjLrn0dZRCToEjrQu7VrwT3nDPC7DBGRJiEhhy2KiMj+FOgiIgGhQBcRCQgFuohIQEQU6GY2yswKzKzQzMaFeb65mb0Yen6umeVEu1AREalfg4FuZqnAeOBMYABwmZnVHlpyLbDNOXc48BDwv9EuVERE6hfJGfpQoNA5t9I5Vw5MBkbXajMaeDr0eArwfdOdC0RE4iqSQO8OrKuxXBRaF7aNc64S2A50iEaBIiISmUguLAp3pl37TsKRtMHMxgJjQ4s7zawggvcPpyOwuZGvTWTJuN/JuM+QnPudjPsMB77fh9b1RCSBXgT0qLGcDayvo02RmTUD2gJba2/IOTcRmBjBe9bLzPKdc3kHu51Ek4z7nYz7DMm538m4zxDd/Y6ky2U+kGtmvcwsHbgUmFqrzVTgR6HHFwP/ds7td4YuIiKx0+AZunOu0sxuAt4CUoEnnXNLzOwBIN85NxWYBDxjZoV4Z+aXxrJoERHZX0STcznnpgPTa627t8bjvcAPoltavQ662yZBJeN+J+M+Q3LudzLuM0Rxv009IyIiwaBL/0VEAkKBLiISEAkX6A3NKxMEZtbDzGaZ2TIzW2Jmt4bWtzezd8zsq9D3wN1zz8xSzWyhmU0LLfcKzQ/0VWi+oHS/a4w2M2tnZlPMbHnomA9PkmN9W+jn+wsze8HMMoJ2vM3sSTPbZGZf1FgX9tia5/+Fsm2xmR1zoO+XUIEe4bwyQVAJ/Nw51x8YBtwY2s9xwLvOuVzg3dBy0NwKLKux/L/AQ6F93oY3b1DQPAzMcM71Awbj7X+gj7WZdQduAfKcc4PwRtBdSvCO91PAqFrr6jq2ZwK5oa+xwIQDfbOECnQim1cm4TnnNjjnPg093oH3H7w7+86Z8zRwvj8VxoaZZQNnA0+Elg04BW9+IAjmPrcBTsAb+otzrtw5V0LAj3VIM6BF6GLElsAGAna8nXMfsP9FlnUd29HAP5xnDtDOzLoeyPslWqBHMq9MoISmIj4amAt0ds5tAC/0gU7+VRYTfwV+AVSHljsAJaH5gSCYx7s3UAz8PdTV9ISZtSLgx9o59zXwJ2AtXpBvBxYQ/OMNdR/bg863RAv0iOaMCQozaw28AvzMOVfqdz2xZGbnAJuccwtqrg7TNGjHuxlwDDDBOXc0sIuAda+EE+o3Hg30AroBrfC6HGoL2vGuz0H/vCdaoEcyr0wgmFkaXpg/55x7NbT6m2//BAt93+RXfTFwPHCema3G60o7Be+MvV3oT3II5vEuAoqcc3NDy1PwAj7IxxrgVGCVc67YOVcBvAp8j+Afb6j72B50viVaoEcyr0zCC/UdTwKWOef+UuOpmnPm/Aj4V7xrixXn3F3OuWznXA7ecf23c+4KYBbe/EAQsH0GcM5tBNaZWd/Qqu8DSwnwsQ5ZCwwzs5ahn/dv9zvQxzukrmM7FfhhaLTLMGD7t10zEXPOJdQXcBbwJbACuNvvemK0jyPw/tRaDHwW+joLr0/5XeCr0Pf2ftcao/0/CZgWetwbmAcUAi8Dzf2uLwb7exSQHzrerwGHJMOxBu4HlgNfAM8AzYN2vIEX8D4jqMA7A7+2rmOL1+UyPpRtn+ONADqg99Ol/yIiAZFoXS4iIlIHBbqISEAo0EVEAkKBLiISEAp0EZGAUKCLiASEAl1EJCD+D0czss0AAAAESURBVBDfFkhYcL7gAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.plot(test_scores)\n",
    "plt.plot(test_scores2)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "理论上，cross entropy只在初期优于MSE，但实际上差距很明显"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
